From 7f834339bae01dbe9898ebaab1dc471196abcfb9 Mon Sep 17 00:00:00 2001 From: Ashwin Bharambe Date: Tue, 5 Aug 2025 14:55:05 -0700 Subject: [PATCH] chore(misc): make tests and starter faster (#3042) A bunch of miscellaneous cleanup focusing on tests, but ended up speeding up starter distro substantially. - Pulled llama stack client init for tests into `pytest_sessionstart` so it does not clobber output - Profiling of that told me where we were doing lots of heavy imports for starter, so lazied them - starter now starts 20seconds+ faster on my Mac - A few other smallish refactors for `compat_client` --- docs/source/providers/agents/index.md | 2 +- docs/source/providers/datasetio/index.md | 2 +- docs/source/providers/eval/index.md | 2 +- docs/source/providers/files/index.md | 2 +- docs/source/providers/files/inline_localfs.md | 2 +- docs/source/providers/inference/index.md | 2 +- .../providers/inference/remote_hf_endpoint.md | 2 +- .../inference/remote_hf_serverless.md | 2 +- docs/source/providers/inference/remote_tgi.md | 2 +- docs/source/providers/post_training/index.md | 2 +- .../post_training/inline_huggingface.md | 3 +- docs/source/providers/safety/index.md | 2 +- docs/source/providers/scoring/index.md | 2 +- docs/source/providers/telemetry/index.md | 2 +- docs/source/providers/tool_runtime/index.md | 2 +- docs/source/providers/vector_io/index.md | 2 +- .../providers/vector_io/inline_chromadb.md | 2 +- .../providers/vector_io/inline_milvus.md | 2 +- .../providers/vector_io/inline_qdrant.md | 2 +- .../providers/vector_io/inline_sqlite-vec.md | 2 +- .../providers/vector_io/inline_sqlite_vec.md | 2 +- .../providers/vector_io/remote_chromadb.md | 2 +- .../providers/vector_io/remote_milvus.md | 4 +- llama_stack/distributions/ci-tests/run.yaml | 1 + llama_stack/distributions/starter/run.yaml | 1 + .../inline/datasetio/localfs/datasetio.py | 6 +- .../post_training/huggingface/config.py | 9 +- .../huggingface/post_training.py | 19 +- .../post_training/torchtune/post_training.py | 11 +- .../datasetio/huggingface/huggingface.py | 6 +- .../remote/inference/ollama/ollama.py | 3 +- .../providers/utils/datasetio/url_utils.py | 4 +- scripts/provider_codegen.py | 7 +- .../agents/test_openai_responses.py | 37 +- tests/integration/conftest.py | 17 + tests/integration/fixtures/common.py | 34 +- .../inference/test_openai_completion.py | 17 - tests/integration/recordings/index.sqlite | Bin 53248 -> 53248 bytes .../recordings/responses/4a3a4447b16b.json | 98 +- .../recordings/responses/b44cc7a7afc8.json | 3076 ++++++++--------- .../recordings/responses/d0ac68cbde69.json | 24 +- .../recordings/responses/da531c71e64f.json | 421 +++ .../recordings/responses/dbc41d2417e1.json | 674 ++++ .../recordings/responses/f1ea938b0b0d.json | 56 + .../vector_io/test_openai_vector_stores.py | 15 - 45 files changed, 2897 insertions(+), 1688 deletions(-) create mode 100644 tests/integration/recordings/responses/da531c71e64f.json create mode 100644 tests/integration/recordings/responses/dbc41d2417e1.json create mode 100644 tests/integration/recordings/responses/f1ea938b0b0d.json diff --git a/docs/source/providers/agents/index.md b/docs/source/providers/agents/index.md index a88f085ad..92bf9edc0 100644 --- a/docs/source/providers/agents/index.md +++ b/docs/source/providers/agents/index.md @@ -1,4 +1,4 @@ -# Agents +# Agents ## Overview diff --git a/docs/source/providers/datasetio/index.md b/docs/source/providers/datasetio/index.md index 9b0f385f4..94a97e2ed 100644 --- a/docs/source/providers/datasetio/index.md +++ b/docs/source/providers/datasetio/index.md @@ -1,4 +1,4 @@ -# Datasetio +# Datasetio ## Overview diff --git a/docs/source/providers/eval/index.md b/docs/source/providers/eval/index.md index f8d24a820..d180d256c 100644 --- a/docs/source/providers/eval/index.md +++ b/docs/source/providers/eval/index.md @@ -1,4 +1,4 @@ -# Eval +# Eval ## Overview diff --git a/docs/source/providers/files/index.md b/docs/source/providers/files/index.md index 8d4f8773a..692aad3ca 100644 --- a/docs/source/providers/files/index.md +++ b/docs/source/providers/files/index.md @@ -1,4 +1,4 @@ -# Files +# Files ## Overview diff --git a/docs/source/providers/files/inline_localfs.md b/docs/source/providers/files/inline_localfs.md index 54c489c7d..09267b7d8 100644 --- a/docs/source/providers/files/inline_localfs.md +++ b/docs/source/providers/files/inline_localfs.md @@ -8,7 +8,7 @@ Local filesystem-based file storage provider for managing files and documents lo | Field | Type | Required | Default | Description | |-------|------|----------|---------|-------------| -| `storage_dir` | `` | No | PydanticUndefined | Directory to store uploaded files | +| `storage_dir` | `` | No | | Directory to store uploaded files | | `metadata_store` | `utils.sqlstore.sqlstore.SqliteSqlStoreConfig \| utils.sqlstore.sqlstore.PostgresSqlStoreConfig` | No | sqlite | SQL store configuration for file metadata | | `ttl_secs` | `` | No | 31536000 | | diff --git a/docs/source/providers/inference/index.md b/docs/source/providers/inference/index.md index 207c28c64..1c7bc86b9 100644 --- a/docs/source/providers/inference/index.md +++ b/docs/source/providers/inference/index.md @@ -1,4 +1,4 @@ -# Inference +# Inference ## Overview diff --git a/docs/source/providers/inference/remote_hf_endpoint.md b/docs/source/providers/inference/remote_hf_endpoint.md index f9ca6b538..8aaf13476 100644 --- a/docs/source/providers/inference/remote_hf_endpoint.md +++ b/docs/source/providers/inference/remote_hf_endpoint.md @@ -8,7 +8,7 @@ HuggingFace Inference Endpoints provider for dedicated model serving. | Field | Type | Required | Default | Description | |-------|------|----------|---------|-------------| -| `endpoint_name` | `` | No | PydanticUndefined | The name of the Hugging Face Inference Endpoint in the format of '{namespace}/{endpoint_name}' (e.g. 'my-cool-org/meta-llama-3-1-8b-instruct-rce'). Namespace is optional and will default to the user account if not provided. | +| `endpoint_name` | `` | No | | The name of the Hugging Face Inference Endpoint in the format of '{namespace}/{endpoint_name}' (e.g. 'my-cool-org/meta-llama-3-1-8b-instruct-rce'). Namespace is optional and will default to the user account if not provided. | | `api_token` | `pydantic.types.SecretStr \| None` | No | | Your Hugging Face user access token (will default to locally saved token if not provided) | ## Sample Configuration diff --git a/docs/source/providers/inference/remote_hf_serverless.md b/docs/source/providers/inference/remote_hf_serverless.md index 345af3e49..6764590b8 100644 --- a/docs/source/providers/inference/remote_hf_serverless.md +++ b/docs/source/providers/inference/remote_hf_serverless.md @@ -8,7 +8,7 @@ HuggingFace Inference API serverless provider for on-demand model inference. | Field | Type | Required | Default | Description | |-------|------|----------|---------|-------------| -| `huggingface_repo` | `` | No | PydanticUndefined | The model ID of the model on the Hugging Face Hub (e.g. 'meta-llama/Meta-Llama-3.1-70B-Instruct') | +| `huggingface_repo` | `` | No | | The model ID of the model on the Hugging Face Hub (e.g. 'meta-llama/Meta-Llama-3.1-70B-Instruct') | | `api_token` | `pydantic.types.SecretStr \| None` | No | | Your Hugging Face user access token (will default to locally saved token if not provided) | ## Sample Configuration diff --git a/docs/source/providers/inference/remote_tgi.md b/docs/source/providers/inference/remote_tgi.md index 125984fab..104bb4aab 100644 --- a/docs/source/providers/inference/remote_tgi.md +++ b/docs/source/providers/inference/remote_tgi.md @@ -8,7 +8,7 @@ Text Generation Inference (TGI) provider for HuggingFace model serving. | Field | Type | Required | Default | Description | |-------|------|----------|---------|-------------| -| `url` | `` | No | PydanticUndefined | The URL for the TGI serving endpoint | +| `url` | `` | No | | The URL for the TGI serving endpoint | ## Sample Configuration diff --git a/docs/source/providers/post_training/index.md b/docs/source/providers/post_training/index.md index fb6af2d57..c6c92c40e 100644 --- a/docs/source/providers/post_training/index.md +++ b/docs/source/providers/post_training/index.md @@ -1,4 +1,4 @@ -# Post_Training +# Post_Training ## Overview diff --git a/docs/source/providers/post_training/inline_huggingface.md b/docs/source/providers/post_training/inline_huggingface.md index 0a8745e71..8b10fe79c 100644 --- a/docs/source/providers/post_training/inline_huggingface.md +++ b/docs/source/providers/post_training/inline_huggingface.md @@ -27,7 +27,7 @@ HuggingFace-based post-training provider for fine-tuning models using the Huggin | `dpo_beta` | `` | No | 0.1 | | | `use_reference_model` | `` | No | True | | | `dpo_loss_type` | `Literal['sigmoid', 'hinge', 'ipo', 'kto_pair'` | No | sigmoid | | -| `dpo_output_dir` | `` | No | ./checkpoints/dpo | | +| `dpo_output_dir` | `` | No | | | ## Sample Configuration @@ -35,6 +35,7 @@ HuggingFace-based post-training provider for fine-tuning models using the Huggin checkpoint_format: huggingface distributed_backend: null device: cpu +dpo_output_dir: ~/.llama/dummy/dpo_output ``` diff --git a/docs/source/providers/safety/index.md b/docs/source/providers/safety/index.md index f82694ac8..5ddda2242 100644 --- a/docs/source/providers/safety/index.md +++ b/docs/source/providers/safety/index.md @@ -1,4 +1,4 @@ -# Safety +# Safety ## Overview diff --git a/docs/source/providers/scoring/index.md b/docs/source/providers/scoring/index.md index 31a87c555..f3bd48eb0 100644 --- a/docs/source/providers/scoring/index.md +++ b/docs/source/providers/scoring/index.md @@ -1,4 +1,4 @@ -# Scoring +# Scoring ## Overview diff --git a/docs/source/providers/telemetry/index.md b/docs/source/providers/telemetry/index.md index 2451e8f62..c7fbfed73 100644 --- a/docs/source/providers/telemetry/index.md +++ b/docs/source/providers/telemetry/index.md @@ -1,4 +1,4 @@ -# Telemetry +# Telemetry ## Overview diff --git a/docs/source/providers/tool_runtime/index.md b/docs/source/providers/tool_runtime/index.md index a0b835e3b..8d29aed43 100644 --- a/docs/source/providers/tool_runtime/index.md +++ b/docs/source/providers/tool_runtime/index.md @@ -1,4 +1,4 @@ -# Tool_Runtime +# Tool_Runtime ## Overview diff --git a/docs/source/providers/vector_io/index.md b/docs/source/providers/vector_io/index.md index a7703ae14..28ae523d7 100644 --- a/docs/source/providers/vector_io/index.md +++ b/docs/source/providers/vector_io/index.md @@ -1,4 +1,4 @@ -# Vector_Io +# Vector_Io ## Overview diff --git a/docs/source/providers/vector_io/inline_chromadb.md b/docs/source/providers/vector_io/inline_chromadb.md index 679c82830..518e3f689 100644 --- a/docs/source/providers/vector_io/inline_chromadb.md +++ b/docs/source/providers/vector_io/inline_chromadb.md @@ -41,7 +41,7 @@ See [Chroma's documentation](https://docs.trychroma.com/docs/overview/introducti | Field | Type | Required | Default | Description | |-------|------|----------|---------|-------------| -| `db_path` | `` | No | PydanticUndefined | | +| `db_path` | `` | No | | | | `kvstore` | `utils.kvstore.config.RedisKVStoreConfig \| utils.kvstore.config.SqliteKVStoreConfig \| utils.kvstore.config.PostgresKVStoreConfig \| utils.kvstore.config.MongoDBKVStoreConfig` | No | sqlite | Config for KV store backend | ## Sample Configuration diff --git a/docs/source/providers/vector_io/inline_milvus.md b/docs/source/providers/vector_io/inline_milvus.md index 3b3aad3fc..33ea4d179 100644 --- a/docs/source/providers/vector_io/inline_milvus.md +++ b/docs/source/providers/vector_io/inline_milvus.md @@ -10,7 +10,7 @@ Please refer to the remote provider documentation. | Field | Type | Required | Default | Description | |-------|------|----------|---------|-------------| -| `db_path` | `` | No | PydanticUndefined | | +| `db_path` | `` | No | | | | `kvstore` | `utils.kvstore.config.RedisKVStoreConfig \| utils.kvstore.config.SqliteKVStoreConfig \| utils.kvstore.config.PostgresKVStoreConfig \| utils.kvstore.config.MongoDBKVStoreConfig` | No | sqlite | Config for KV store backend (SQLite only for now) | | `consistency_level` | `` | No | Strong | The consistency level of the Milvus server | diff --git a/docs/source/providers/vector_io/inline_qdrant.md b/docs/source/providers/vector_io/inline_qdrant.md index e989a3554..b5072d220 100644 --- a/docs/source/providers/vector_io/inline_qdrant.md +++ b/docs/source/providers/vector_io/inline_qdrant.md @@ -50,7 +50,7 @@ See the [Qdrant documentation](https://qdrant.tech/documentation/) for more deta | Field | Type | Required | Default | Description | |-------|------|----------|---------|-------------| -| `path` | `` | No | PydanticUndefined | | +| `path` | `` | No | | | | `kvstore` | `utils.kvstore.config.RedisKVStoreConfig \| utils.kvstore.config.SqliteKVStoreConfig \| utils.kvstore.config.PostgresKVStoreConfig \| utils.kvstore.config.MongoDBKVStoreConfig` | No | sqlite | | ## Sample Configuration diff --git a/docs/source/providers/vector_io/inline_sqlite-vec.md b/docs/source/providers/vector_io/inline_sqlite-vec.md index ae7c45b21..854bb9d08 100644 --- a/docs/source/providers/vector_io/inline_sqlite-vec.md +++ b/docs/source/providers/vector_io/inline_sqlite-vec.md @@ -205,7 +205,7 @@ See [sqlite-vec's GitHub repo](https://github.com/asg017/sqlite-vec/tree/main) f | Field | Type | Required | Default | Description | |-------|------|----------|---------|-------------| -| `db_path` | `` | No | PydanticUndefined | Path to the SQLite database file | +| `db_path` | `` | No | | Path to the SQLite database file | | `kvstore` | `utils.kvstore.config.RedisKVStoreConfig \| utils.kvstore.config.SqliteKVStoreConfig \| utils.kvstore.config.PostgresKVStoreConfig \| utils.kvstore.config.MongoDBKVStoreConfig` | No | sqlite | Config for KV store backend (SQLite only for now) | ## Sample Configuration diff --git a/docs/source/providers/vector_io/inline_sqlite_vec.md b/docs/source/providers/vector_io/inline_sqlite_vec.md index 7e14bb8bd..7ad8eb252 100644 --- a/docs/source/providers/vector_io/inline_sqlite_vec.md +++ b/docs/source/providers/vector_io/inline_sqlite_vec.md @@ -10,7 +10,7 @@ Please refer to the sqlite-vec provider documentation. | Field | Type | Required | Default | Description | |-------|------|----------|---------|-------------| -| `db_path` | `` | No | PydanticUndefined | Path to the SQLite database file | +| `db_path` | `` | No | | Path to the SQLite database file | | `kvstore` | `utils.kvstore.config.RedisKVStoreConfig \| utils.kvstore.config.SqliteKVStoreConfig \| utils.kvstore.config.PostgresKVStoreConfig \| utils.kvstore.config.MongoDBKVStoreConfig` | No | sqlite | Config for KV store backend (SQLite only for now) | ## Sample Configuration diff --git a/docs/source/providers/vector_io/remote_chromadb.md b/docs/source/providers/vector_io/remote_chromadb.md index 447ea6cd6..badfebe90 100644 --- a/docs/source/providers/vector_io/remote_chromadb.md +++ b/docs/source/providers/vector_io/remote_chromadb.md @@ -40,7 +40,7 @@ See [Chroma's documentation](https://docs.trychroma.com/docs/overview/introducti | Field | Type | Required | Default | Description | |-------|------|----------|---------|-------------| -| `url` | `str \| None` | No | PydanticUndefined | | +| `url` | `str \| None` | No | | | | `kvstore` | `utils.kvstore.config.RedisKVStoreConfig \| utils.kvstore.config.SqliteKVStoreConfig \| utils.kvstore.config.PostgresKVStoreConfig \| utils.kvstore.config.MongoDBKVStoreConfig` | No | sqlite | Config for KV store backend | ## Sample Configuration diff --git a/docs/source/providers/vector_io/remote_milvus.md b/docs/source/providers/vector_io/remote_milvus.md index 6734d8315..3646f4acc 100644 --- a/docs/source/providers/vector_io/remote_milvus.md +++ b/docs/source/providers/vector_io/remote_milvus.md @@ -111,8 +111,8 @@ For more details on TLS configuration, refer to the [TLS setup guide](https://mi | Field | Type | Required | Default | Description | |-------|------|----------|---------|-------------| -| `uri` | `` | No | PydanticUndefined | The URI of the Milvus server | -| `token` | `str \| None` | No | PydanticUndefined | The token of the Milvus server | +| `uri` | `` | No | | The URI of the Milvus server | +| `token` | `str \| None` | No | | The token of the Milvus server | | `consistency_level` | `` | No | Strong | The consistency level of the Milvus server | | `kvstore` | `utils.kvstore.config.RedisKVStoreConfig \| utils.kvstore.config.SqliteKVStoreConfig \| utils.kvstore.config.PostgresKVStoreConfig \| utils.kvstore.config.MongoDBKVStoreConfig` | No | sqlite | Config for KV store backend | | `config` | `dict` | No | {} | This configuration allows additional fields to be passed through to the underlying Milvus client. See the [Milvus](https://milvus.io/docs/install-overview.md) documentation for more details about Milvus in general. | diff --git a/llama_stack/distributions/ci-tests/run.yaml b/llama_stack/distributions/ci-tests/run.yaml index becec81c6..188c66275 100644 --- a/llama_stack/distributions/ci-tests/run.yaml +++ b/llama_stack/distributions/ci-tests/run.yaml @@ -154,6 +154,7 @@ providers: checkpoint_format: huggingface distributed_backend: null device: cpu + dpo_output_dir: ~/.llama/distributions/ci-tests/dpo_output eval: - provider_id: meta-reference provider_type: inline::meta-reference diff --git a/llama_stack/distributions/starter/run.yaml b/llama_stack/distributions/starter/run.yaml index d56559ebc..8bd737686 100644 --- a/llama_stack/distributions/starter/run.yaml +++ b/llama_stack/distributions/starter/run.yaml @@ -154,6 +154,7 @@ providers: checkpoint_format: huggingface distributed_backend: null device: cpu + dpo_output_dir: ~/.llama/distributions/starter/dpo_output eval: - provider_id: meta-reference provider_type: inline::meta-reference diff --git a/llama_stack/providers/inline/datasetio/localfs/datasetio.py b/llama_stack/providers/inline/datasetio/localfs/datasetio.py index da71ecb17..e8ebeb30d 100644 --- a/llama_stack/providers/inline/datasetio/localfs/datasetio.py +++ b/llama_stack/providers/inline/datasetio/localfs/datasetio.py @@ -5,8 +5,6 @@ # the root directory of this source tree. from typing import Any -import pandas - from llama_stack.apis.common.responses import PaginatedResponse from llama_stack.apis.datasetio import DatasetIO from llama_stack.apis.datasets import Dataset @@ -44,6 +42,8 @@ class PandasDataframeDataset: if self.dataset_def.source.type == "uri": self.df = await get_dataframe_from_uri(self.dataset_def.source.uri) elif self.dataset_def.source.type == "rows": + import pandas + self.df = pandas.DataFrame(self.dataset_def.source.rows) else: raise ValueError(f"Unsupported dataset source type: {self.dataset_def.source.type}") @@ -103,6 +103,8 @@ class LocalFSDatasetIOImpl(DatasetIO, DatasetsProtocolPrivate): return paginate_records(records, start_index, limit) async def append_rows(self, dataset_id: str, rows: list[dict[str, Any]]) -> None: + import pandas + dataset_def = self.dataset_infos[dataset_id] dataset_impl = PandasDataframeDataset(dataset_def) await dataset_impl.load() diff --git a/llama_stack/providers/inline/post_training/huggingface/config.py b/llama_stack/providers/inline/post_training/huggingface/config.py index dae8fcc04..04e286ff0 100644 --- a/llama_stack/providers/inline/post_training/huggingface/config.py +++ b/llama_stack/providers/inline/post_training/huggingface/config.py @@ -71,8 +71,13 @@ class HuggingFacePostTrainingConfig(BaseModel): dpo_beta: float = 0.1 use_reference_model: bool = True dpo_loss_type: Literal["sigmoid", "hinge", "ipo", "kto_pair"] = "sigmoid" - dpo_output_dir: str = "./checkpoints/dpo" + dpo_output_dir: str @classmethod def sample_run_config(cls, __distro_dir__: str, **kwargs: Any) -> dict[str, Any]: - return {"checkpoint_format": "huggingface", "distributed_backend": None, "device": "cpu"} + return { + "checkpoint_format": "huggingface", + "distributed_backend": None, + "device": "cpu", + "dpo_output_dir": __distro_dir__ + "/dpo_output", + } diff --git a/llama_stack/providers/inline/post_training/huggingface/post_training.py b/llama_stack/providers/inline/post_training/huggingface/post_training.py index 81622e2b7..22ace1ae0 100644 --- a/llama_stack/providers/inline/post_training/huggingface/post_training.py +++ b/llama_stack/providers/inline/post_training/huggingface/post_training.py @@ -22,15 +22,8 @@ from llama_stack.apis.post_training import ( from llama_stack.providers.inline.post_training.huggingface.config import ( HuggingFacePostTrainingConfig, ) -from llama_stack.providers.inline.post_training.huggingface.recipes.finetune_single_device import ( - HFFinetuningSingleDevice, -) -from llama_stack.providers.inline.post_training.huggingface.recipes.finetune_single_device_dpo import ( - HFDPOAlignmentSingleDevice, -) from llama_stack.providers.utils.scheduler import JobArtifact, Scheduler from llama_stack.providers.utils.scheduler import JobStatus as SchedulerJobStatus -from llama_stack.schema_utils import webmethod class TrainingArtifactType(Enum): @@ -85,6 +78,10 @@ class HuggingFacePostTrainingImpl: algorithm_config: AlgorithmConfig | None = None, ) -> PostTrainingJob: async def handler(on_log_message_cb, on_status_change_cb, on_artifact_collected_cb): + from llama_stack.providers.inline.post_training.huggingface.recipes.finetune_single_device import ( + HFFinetuningSingleDevice, + ) + on_log_message_cb("Starting HF finetuning") recipe = HFFinetuningSingleDevice( @@ -124,6 +121,10 @@ class HuggingFacePostTrainingImpl: logger_config: dict[str, Any], ) -> PostTrainingJob: async def handler(on_log_message_cb, on_status_change_cb, on_artifact_collected_cb): + from llama_stack.providers.inline.post_training.huggingface.recipes.finetune_single_device_dpo import ( + HFDPOAlignmentSingleDevice, + ) + on_log_message_cb("Starting HF DPO alignment") recipe = HFDPOAlignmentSingleDevice( @@ -168,7 +169,6 @@ class HuggingFacePostTrainingImpl: data = cls._get_artifacts_metadata_by_type(job, TrainingArtifactType.RESOURCES_STATS.value) return data[0] if data else None - @webmethod(route="/post-training/job/status") async def get_training_job_status(self, job_uuid: str) -> PostTrainingJobStatusResponse | None: job = self._scheduler.get_job(job_uuid) @@ -195,16 +195,13 @@ class HuggingFacePostTrainingImpl: resources_allocated=self._get_resources_allocated(job), ) - @webmethod(route="/post-training/job/cancel") async def cancel_training_job(self, job_uuid: str) -> None: self._scheduler.cancel(job_uuid) - @webmethod(route="/post-training/job/artifacts") async def get_training_job_artifacts(self, job_uuid: str) -> PostTrainingJobArtifactsResponse | None: job = self._scheduler.get_job(job_uuid) return PostTrainingJobArtifactsResponse(job_uuid=job_uuid, checkpoints=self._get_checkpoints(job)) - @webmethod(route="/post-training/jobs", method="GET") async def get_training_jobs(self) -> ListPostTrainingJobsResponse: return ListPostTrainingJobsResponse( data=[PostTrainingJob(job_uuid=job.id) for job in self._scheduler.get_jobs()] diff --git a/llama_stack/providers/inline/post_training/torchtune/post_training.py b/llama_stack/providers/inline/post_training/torchtune/post_training.py index d20e11b11..765f6789d 100644 --- a/llama_stack/providers/inline/post_training/torchtune/post_training.py +++ b/llama_stack/providers/inline/post_training/torchtune/post_training.py @@ -23,12 +23,8 @@ from llama_stack.apis.post_training import ( from llama_stack.providers.inline.post_training.torchtune.config import ( TorchtunePostTrainingConfig, ) -from llama_stack.providers.inline.post_training.torchtune.recipes.lora_finetuning_single_device import ( - LoraFinetuningSingleDevice, -) from llama_stack.providers.utils.scheduler import JobArtifact, Scheduler from llama_stack.providers.utils.scheduler import JobStatus as SchedulerJobStatus -from llama_stack.schema_utils import webmethod class TrainingArtifactType(Enum): @@ -84,6 +80,10 @@ class TorchtunePostTrainingImpl: if isinstance(algorithm_config, LoraFinetuningConfig): async def handler(on_log_message_cb, on_status_change_cb, on_artifact_collected_cb): + from llama_stack.providers.inline.post_training.torchtune.recipes.lora_finetuning_single_device import ( + LoraFinetuningSingleDevice, + ) + on_log_message_cb("Starting Lora finetuning") recipe = LoraFinetuningSingleDevice( @@ -144,7 +144,6 @@ class TorchtunePostTrainingImpl: data = cls._get_artifacts_metadata_by_type(job, TrainingArtifactType.RESOURCES_STATS.value) return data[0] if data else None - @webmethod(route="/post-training/job/status") async def get_training_job_status(self, job_uuid: str) -> PostTrainingJobStatusResponse | None: job = self._scheduler.get_job(job_uuid) @@ -171,11 +170,9 @@ class TorchtunePostTrainingImpl: resources_allocated=self._get_resources_allocated(job), ) - @webmethod(route="/post-training/job/cancel") async def cancel_training_job(self, job_uuid: str) -> None: self._scheduler.cancel(job_uuid) - @webmethod(route="/post-training/job/artifacts") async def get_training_job_artifacts(self, job_uuid: str) -> PostTrainingJobArtifactsResponse | None: job = self._scheduler.get_job(job_uuid) return PostTrainingJobArtifactsResponse(job_uuid=job_uuid, checkpoints=self._get_checkpoints(job)) diff --git a/llama_stack/providers/remote/datasetio/huggingface/huggingface.py b/llama_stack/providers/remote/datasetio/huggingface/huggingface.py index fafd1d8ff..a34e354bf 100644 --- a/llama_stack/providers/remote/datasetio/huggingface/huggingface.py +++ b/llama_stack/providers/remote/datasetio/huggingface/huggingface.py @@ -6,8 +6,6 @@ from typing import Any from urllib.parse import parse_qs, urlparse -import datasets as hf_datasets - from llama_stack.apis.common.responses import PaginatedResponse from llama_stack.apis.datasetio import DatasetIO from llama_stack.apis.datasets import Dataset @@ -73,6 +71,8 @@ class HuggingfaceDatasetIOImpl(DatasetIO, DatasetsProtocolPrivate): start_index: int | None = None, limit: int | None = None, ) -> PaginatedResponse: + import datasets as hf_datasets + dataset_def = self.dataset_infos[dataset_id] path, params = parse_hf_params(dataset_def) loaded_dataset = hf_datasets.load_dataset(path, **params) @@ -81,6 +81,8 @@ class HuggingfaceDatasetIOImpl(DatasetIO, DatasetsProtocolPrivate): return paginate_records(records, start_index, limit) async def append_rows(self, dataset_id: str, rows: list[dict[str, Any]]) -> None: + import datasets as hf_datasets + dataset_def = self.dataset_infos[dataset_id] path, params = parse_hf_params(dataset_def) loaded_dataset = hf_datasets.load_dataset(path, **params) diff --git a/llama_stack/providers/remote/inference/ollama/ollama.py b/llama_stack/providers/remote/inference/ollama/ollama.py index 098e4d324..26b4dec76 100644 --- a/llama_stack/providers/remote/inference/ollama/ollama.py +++ b/llama_stack/providers/remote/inference/ollama/ollama.py @@ -112,7 +112,8 @@ class OllamaInferenceAdapter( @property def openai_client(self) -> AsyncOpenAI: if self._openai_client is None: - self._openai_client = AsyncOpenAI(base_url=f"{self.config.url}/v1", api_key="ollama") + url = self.config.url.rstrip("/") + self._openai_client = AsyncOpenAI(base_url=f"{url}/v1", api_key="ollama") return self._openai_client async def initialize(self) -> None: diff --git a/llama_stack/providers/utils/datasetio/url_utils.py b/llama_stack/providers/utils/datasetio/url_utils.py index 386ee736d..77b047e2d 100644 --- a/llama_stack/providers/utils/datasetio/url_utils.py +++ b/llama_stack/providers/utils/datasetio/url_utils.py @@ -9,12 +9,12 @@ import base64 import io from urllib.parse import unquote -import pandas - from llama_stack.providers.utils.memory.vector_store import parse_data_url async def get_dataframe_from_uri(uri: str): + import pandas + df = None if uri.endswith(".csv"): # Moving to its own thread to avoid io from blocking the eventloop diff --git a/scripts/provider_codegen.py b/scripts/provider_codegen.py index 80c5b7840..84c45fe27 100755 --- a/scripts/provider_codegen.py +++ b/scripts/provider_codegen.py @@ -10,6 +10,7 @@ import sys from pathlib import Path from typing import Any +from pydantic_core import PydanticUndefined from rich.progress import Progress, SpinnerColumn, TextColumn from llama_stack.core.distribution import get_provider_registry @@ -59,6 +60,8 @@ def get_config_class_info(config_class_path: str) -> dict[str, Any]: if hasattr(config_class, "model_fields"): for field_name, field in config_class.model_fields.items(): field_type = str(field.annotation) if field.annotation else "Any" + + # this string replace is ridiculous field_type = field_type.replace("typing.", "").replace("Optional[", "").replace("]", "") field_type = field_type.replace("Annotated[", "").replace("FieldInfo(", "").replace(")", "") field_type = field_type.replace("llama_stack.apis.inference.inference.", "") @@ -77,7 +80,7 @@ def get_config_class_info(config_class_path: str) -> dict[str, Any]: default_value = f"~/.llama/{path_part}" except Exception: default_value = "" - elif field.default is None: + elif field.default is None or field.default is PydanticUndefined: default_value = "" field_info = { @@ -255,7 +258,7 @@ def process_provider_registry(progress, change_tracker: ChangedPathTracker) -> N change_tracker.add_paths(doc_output_dir) index_content = [] - index_content.append(f"# {api_name.title()} \n") + index_content.append(f"# {api_name.title()}\n") index_content.append("## Overview\n") index_content.append( diff --git a/tests/integration/agents/test_openai_responses.py b/tests/integration/agents/test_openai_responses.py index 784ab6893..c783cf99b 100644 --- a/tests/integration/agents/test_openai_responses.py +++ b/tests/integration/agents/test_openai_responses.py @@ -9,12 +9,6 @@ from openai import BadRequestError, OpenAI from llama_stack.core.library_client import LlamaStackAsLibraryClient -@pytest.fixture -def openai_client(client_with_models): - base_url = f"{client_with_models.base_url}/v1/openai/v1" - return OpenAI(base_url=base_url, api_key="bar") - - @pytest.mark.parametrize( "stream", [ @@ -41,15 +35,14 @@ def openai_client(client_with_models): ], ], ) -def test_responses_store(openai_client, client_with_models, text_model_id, stream, tools): - if isinstance(client_with_models, LlamaStackAsLibraryClient): - pytest.skip("OpenAI responses are not supported when testing with library client yet.") +def test_responses_store(compat_client, text_model_id, stream, tools): + if not isinstance(compat_client, OpenAI): + pytest.skip("OpenAI client is required until responses.delete() exists in llama-stack-client") - client = openai_client message = "What's the weather in Tokyo?" + ( " YOU MUST USE THE get_weather function to get the weather." if tools else "" ) - response = client.responses.create( + response = compat_client.responses.create( model=text_model_id, input=[ { @@ -78,14 +71,8 @@ def test_responses_store(openai_client, client_with_models, text_model_id, strea if output_type == "message": content = response.output[0].content[0].text - # list responses - use the underlying HTTP client for endpoints not in SDK - list_response = client._client.get("/responses") - assert list_response.status_code == 200 - data = list_response.json()["data"] - assert response_id in [r["id"] for r in data] - # test retrieve response - retrieved_response = client.responses.retrieve(response_id) + retrieved_response = compat_client.responses.retrieve(response_id) assert retrieved_response.id == response_id assert retrieved_response.model == text_model_id assert retrieved_response.output[0].type == output_type, retrieved_response @@ -93,23 +80,19 @@ def test_responses_store(openai_client, client_with_models, text_model_id, strea assert retrieved_response.output[0].content[0].text == content # Delete the response - delete_response = client.responses.delete(response_id) + delete_response = compat_client.responses.delete(response_id) assert delete_response is None with pytest.raises(BadRequestError): - client.responses.retrieve(response_id) + compat_client.responses.retrieve(response_id) -def test_list_response_input_items(openai_client, client_with_models, text_model_id): +def test_list_response_input_items(compat_client, text_model_id): """Test the new list_openai_response_input_items endpoint.""" - if isinstance(client_with_models, LlamaStackAsLibraryClient): - pytest.skip("OpenAI responses are not supported when testing with library client yet.") - - client = openai_client message = "What is the capital of France?" # Create a response first - response = client.responses.create( + response = compat_client.responses.create( model=text_model_id, input=[ { @@ -123,7 +106,7 @@ def test_list_response_input_items(openai_client, client_with_models, text_model response_id = response.id # Test the new list input items endpoint - input_items_response = client.responses.input_items.list(response_id=response_id) + input_items_response = compat_client.responses.input_items.list(response_id=response_id) # Verify the structure follows OpenAI API spec assert input_items_response.object == "list" diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index daf80059c..9c30d984f 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -9,12 +9,15 @@ import os import platform import textwrap import time +import warnings import pytest from dotenv import load_dotenv from llama_stack.log import get_logger +from .fixtures.common import instantiate_llama_stack_client + logger = get_logger(__name__, category="tests") @@ -27,6 +30,20 @@ def pytest_runtest_makereport(item, call): item.was_xfail = getattr(report, "wasxfail", False) +def pytest_sessionstart(session): + # stop macOS from complaining about duplicate OpenMP libraries + os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE" + + # pull client instantiation to session start so all the complex logs during initialization + # don't clobber the test one-liner outputs + print("instantiating llama_stack_client") + start_time = time.time() + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=DeprecationWarning) + session._llama_stack_client = instantiate_llama_stack_client(session) + print(f"llama_stack_client instantiated in {time.time() - start_time:.3f}s") + + def pytest_runtest_teardown(item): # Check if the test actually ran and passed or failed, but was not skipped or an expected failure (xfail) outcome = getattr(item, "execution_outcome", None) diff --git a/tests/integration/fixtures/common.py b/tests/integration/fixtures/common.py index b9e0512ce..4549a2fc2 100644 --- a/tests/integration/fixtures/common.py +++ b/tests/integration/fixtures/common.py @@ -82,8 +82,7 @@ def wait_for_server_ready(base_url: str, timeout: int = 30, process: subprocess. return False -@pytest.fixture(scope="session") -def provider_data(): +def get_provider_data(): # TODO: this needs to be generalized so each provider can have a sample provider data just # like sample run config on which we can do replace_env_vars() keymap = { @@ -178,8 +177,14 @@ def skip_if_no_model(request): @pytest.fixture(scope="session") -def llama_stack_client(request, provider_data): - config = request.config.getoption("--stack-config") +def llama_stack_client(request): + client = request.session._llama_stack_client + assert client is not None, "llama_stack_client not found in session cache" + return client + + +def instantiate_llama_stack_client(session): + config = session.config.getoption("--stack-config") if not config: config = get_env_or_fail("LLAMA_STACK_CONFIG") @@ -212,13 +217,13 @@ def llama_stack_client(request, provider_data): print(f"Server is ready at {base_url}") # Store process for potential cleanup (pytest will handle termination at session end) - request.session._llama_stack_server_process = server_process + session._llama_stack_server_process = server_process else: print(f"Port {port} is already in use, assuming server is already running...") return LlamaStackClient( base_url=base_url, - provider_data=provider_data, + provider_data=get_provider_data(), timeout=int(os.environ.get("LLAMA_STACK_CLIENT_TIMEOUT", "30")), ) @@ -228,7 +233,7 @@ def llama_stack_client(request, provider_data): if parsed_url.scheme and parsed_url.netloc: return LlamaStackClient( base_url=config, - provider_data=provider_data, + provider_data=get_provider_data(), ) except Exception: # If URL parsing fails, treat as non-URL config @@ -243,7 +248,7 @@ def llama_stack_client(request, provider_data): client = LlamaStackAsLibraryClient( config, - provider_data=provider_data, + provider_data=get_provider_data(), skip_logger_removal=True, ) if not client.initialize(): @@ -258,8 +263,17 @@ def openai_client(client_with_models): return OpenAI(base_url=base_url, api_key="fake") -@pytest.fixture(params=["openai_client", "llama_stack_client"]) -def compat_client(request): +@pytest.fixture(params=["openai_client", "client_with_models"]) +def compat_client(request, client_with_models): + if isinstance(client_with_models, LlamaStackAsLibraryClient): + # OpenAI client expects a server, so unless we also rewrite OpenAI client's requests + # to go via the Stack library client (which itself rewrites requests to be served inline), + # we cannot do this. + # + # This means when we are using Stack as a library, we will test only via the Llama Stack client. + # When we are using a server setup, we can exercise both OpenAI and Llama Stack clients. + pytest.skip("(OpenAI) Compat client cannot be used with Stack library client") + return request.getfixturevalue(request.param) diff --git a/tests/integration/inference/test_openai_completion.py b/tests/integration/inference/test_openai_completion.py index 9927b6ee7..0222bfb79 100644 --- a/tests/integration/inference/test_openai_completion.py +++ b/tests/integration/inference/test_openai_completion.py @@ -6,9 +6,6 @@ import pytest -from openai import OpenAI - -from llama_stack.core.library_client import LlamaStackAsLibraryClient from ..test_cases.test_case import TestCase @@ -59,9 +56,6 @@ def skip_if_model_doesnt_support_suffix(client_with_models, model_id): def skip_if_model_doesnt_support_openai_chat_completion(client_with_models, model_id): - if isinstance(client_with_models, LlamaStackAsLibraryClient): - pytest.skip("OpenAI chat completions are not supported when testing with library client yet.") - provider = provider_from_model(client_with_models, model_id) if provider.provider_type in ( "inline::meta-reference", @@ -90,17 +84,6 @@ def skip_if_provider_isnt_openai(client_with_models, model_id): ) -@pytest.fixture -def openai_client(client_with_models): - base_url = f"{client_with_models.base_url}/v1/openai/v1" - return OpenAI(base_url=base_url, api_key="bar") - - -@pytest.fixture(params=["openai_client", "llama_stack_client"]) -def compat_client(request): - return request.getfixturevalue(request.param) - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/recordings/index.sqlite b/tests/integration/recordings/index.sqlite index cec7df5b05b832fdf9d7f84f5c679c720b473656..e3b42ae9ddb60d50cf3d53987f9fbf1beabc374d 100644 GIT binary patch delta 543 zcmZozz}&Ead4d!ZtMEh_Cm^{oVPQNItNvu>1Zy#tg*8uu7K$@&j+wfYi-j?VL2@$d0z)>2 zYYYwy4x4Qj7%;LhFfgWVEKFkDd})ypFS9VK%VhQqvRrIQY`3@zSTD1>Y%COGjcDZV z65?p&)z{WF=1562H8xB(H%v7%NlQ*KF-T2LG&Zs{NU}^!OioHkN=-~OGDuA|F*h_f zF)*?)wMaEIF*P?xu}HHtOEEA^Gep)uIdQs{poOl1se+NAm4T_1ks;6~!O0h=)Cr)7 zmUh-o&R>|#3sGlcWnf;~#WMN%!fZach()7NhwJ1yLameM@8sG1e0vu≻_D&CH?U=r^mNDLD zW7!;%u%DTUW#MGQRIkYnZVoI=#;p24j`!quDUM8>fYXKPZ+Lkwpn1n$Xwc?Fu7VFL%>4Uz(m2w$ja2j%G98=LuK;aZP|PX5e5c^ zvWbO7n=dUg;$`M%51P!rL6(WTU}K>UJ4>TbhwJ1yLameM@8sG1e0vu&S4RLxBd@+R zuQA7D`?C&{ckfo@f*56HWnj7a*={)&E*TCR2L7G=Mf}>61r&@nI~~}}KC!`bGJhWH ZX2%nDtV}Ecn;XxXF)=ZlOx}6k9RTWtX^a2> diff --git a/tests/integration/recordings/responses/4a3a4447b16b.json b/tests/integration/recordings/responses/4a3a4447b16b.json index fbc09818b..dbaec07e9 100644 --- a/tests/integration/recordings/responses/4a3a4447b16b.json +++ b/tests/integration/recordings/responses/4a3a4447b16b.json @@ -14,7 +14,7 @@ "models": [ { "model": "nomic-embed-text:latest", - "modified_at": "2025-08-04T15:54:50.584357-07:00", + "modified_at": "2025-08-05T14:04:07.946926-07:00", "digest": "0a109f422b47e3a30ba2b10eca18548e944e8a23073ee3f3e947efcf3c45e59f", "size": 274302450, "details": { @@ -28,9 +28,41 @@ "quantization_level": "F16" } }, + { + "model": "llama3.2-vision:11b", + "modified_at": "2025-07-30T18:45:02.517873-07:00", + "digest": "6f2f9757ae97e8a3f8ea33d6adb2b11d93d9a35bef277cd2c0b1b5af8e8d0b1e", + "size": 7816589186, + "details": { + "parent_model": "", + "format": "gguf", + "family": "mllama", + "families": [ + "mllama" + ], + "parameter_size": "10.7B", + "quantization_level": "Q4_K_M" + } + }, + { + "model": "llama3.2-vision:latest", + "modified_at": "2025-07-29T20:18:47.920468-07:00", + "digest": "6f2f9757ae97e8a3f8ea33d6adb2b11d93d9a35bef277cd2c0b1b5af8e8d0b1e", + "size": 7816589186, + "details": { + "parent_model": "", + "format": "gguf", + "family": "mllama", + "families": [ + "mllama" + ], + "parameter_size": "10.7B", + "quantization_level": "Q4_K_M" + } + }, { "model": "llama-guard3:1b", - "modified_at": "2025-08-01T15:46:28.963517-07:00", + "modified_at": "2025-07-25T14:39:44.978630-07:00", "digest": "494147e06bf99e10dbe67b63a07ac81c162f18ef3341aa3390007ac828571b3b", "size": 1600181919, "details": { @@ -46,7 +78,7 @@ }, { "model": "all-minilm:l6-v2", - "modified_at": "2025-07-29T15:07:06.295748-07:00", + "modified_at": "2025-07-24T15:15:11.129290-07:00", "digest": "1b226e2802dbb772b5fc32a58f103ca1804ef7501331012de126ab22f67475ef", "size": 45960996, "details": { @@ -61,26 +93,10 @@ } }, { - "model": "all-minilm:latest", - "modified_at": "2025-06-04T12:06:43.990073-07:00", - "digest": "1b226e2802dbb772b5fc32a58f103ca1804ef7501331012de126ab22f67475ef", - "size": 45960996, - "details": { - "parent_model": "", - "format": "gguf", - "family": "bert", - "families": [ - "bert" - ], - "parameter_size": "23M", - "quantization_level": "F16" - } - }, - { - "model": "llama3.1:8b-instruct-fp16", - "modified_at": "2025-02-14T15:23:24.865395-08:00", - "digest": "4aacac4194543ff7f70dab3f2ebc169c132d5319bb36f7a7e99c4ff525ebcc09", - "size": 16068910253, + "model": "llama3.2:1b", + "modified_at": "2025-07-17T22:02:24.953208-07:00", + "digest": "baf6a787fdffd633537aa2eb51cfd54cb93ff08e28040095462bb63daf552878", + "size": 1321098329, "details": { "parent_model": "", "format": "gguf", @@ -88,13 +104,45 @@ "families": [ "llama" ], - "parameter_size": "8.0B", + "parameter_size": "1.2B", + "quantization_level": "Q8_0" + } + }, + { + "model": "all-minilm:latest", + "modified_at": "2025-06-03T16:50:10.946583-07:00", + "digest": "1b226e2802dbb772b5fc32a58f103ca1804ef7501331012de126ab22f67475ef", + "size": 45960996, + "details": { + "parent_model": "", + "format": "gguf", + "family": "bert", + "families": [ + "bert" + ], + "parameter_size": "23M", "quantization_level": "F16" } }, + { + "model": "llama3.2:3b", + "modified_at": "2025-05-01T11:15:23.797447-07:00", + "digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72", + "size": 2019393189, + "details": { + "parent_model": "", + "format": "gguf", + "family": "llama", + "families": [ + "llama" + ], + "parameter_size": "3.2B", + "quantization_level": "Q4_K_M" + } + }, { "model": "llama3.2:3b-instruct-fp16", - "modified_at": "2025-01-21T13:46:43.514008-08:00", + "modified_at": "2025-04-30T15:33:48.939665-07:00", "digest": "195a8c01d91ec3cb1e0aad4624a51f2602c51fa7d96110f8ab5a20c84081804d", "size": 6433703586, "details": { diff --git a/tests/integration/recordings/responses/b44cc7a7afc8.json b/tests/integration/recordings/responses/b44cc7a7afc8.json index dc770b693..2dbbf7801 100644 --- a/tests/integration/recordings/responses/b44cc7a7afc8.json +++ b/tests/integration/recordings/responses/b44cc7a7afc8.json @@ -23,1556 +23,1556 @@ "created_at": null, "done": null, "done_reason": null, - "total_duration": 91561401, - "load_duration": 7565647, + "total_duration": 105611084, + "load_duration": 42126542, "prompt_eval_count": 162, "prompt_eval_duration": null, "eval_count": null, "eval_duration": null, "embeddings": [ [ - -0.074518315, - 0.027976887, - -0.02593837, - 0.028388586, - -0.04876724, - -0.124505915, - -0.03774251, - 0.041209254, - -0.048753787, - -0.027770191, - -0.092701025, - 0.051893745, - 0.080871776, - 0.023072483, - 0.103199854, - -0.061396316, - -0.046632618, - 0.031259038, - -0.009099142, - -0.110972114, - -0.020746639, - 0.036444016, - -0.013380681, - 0.007444351, - 0.01946104, - 0.0044101896, - -0.012363551, - -0.044742297, - -0.000109245026, - 0.027794957, - -0.05248249, - 0.062051035, - 0.019644303, - 0.02233988, - -0.018772174, - 0.06638057, - -0.03694357, - -0.09439974, - -0.0498964, - -0.016085815, - -0.08936965, - 0.07279298, - -0.073282845, - -0.027557392, - -0.0663739, - 0.015465914, - -0.004212829, - -0.03255469, - -0.035763785, - -0.026250161, - -0.09131691, - 0.022791812, - -0.0560323, - -0.103517555, - -0.02337786, - -0.016169889, - 0.06033412, - -0.012003445, - -0.009792255, - -0.1520526, - -0.07313599, - 0.022364737, - 0.08799007, - 0.062190924, - -0.0476343, - -0.055481553, - -0.036080837, - 0.01777673, - 0.084963925, - -0.050720915, - -0.09959311, - -0.029466882, - -0.0020879637, - 0.08149215, - 0.030555075, - -0.12159375, - 0.04638196, - 0.0054617906, - -0.007600725, - 0.04925988, - 0.0010117136, - 0.01823397, - -0.056374155, - 0.0908255, - 0.03542638, - -0.06282811, - 0.05174182, - 0.07176561, - -0.04539055, - 0.009702367, - 0.03031262, - -0.05891284, - 0.037203796, - 0.0041589597, - -0.014310235, - 0.062144686, - 0.026470749, - -0.054097973, - -0.040584724, - 0.010875258, - -0.003349861, - -0.077305436, - 0.055475105, - 0.011331311, - 0.049933966, - 0.011079793, - 0.04419192, - -0.088725075, - 0.04790246, - -0.029256914, - -0.021242525, - -0.059049267, - -0.0059888517, - -0.036285046, - 0.045748435, - 0.07392407, - -0.0042937263, - 0.07591468, - -0.00059363164, - 0.006329638, - 0.019841122, - -0.008888848, - -0.0074318657, - -0.014973693, - 0.012456981, - 0.0033115426, - -0.009930274, - 1.5269222e-33, - -0.03017857, - -0.056806926, - -0.009980652, - 0.022316255, - 0.057149988, - -0.01857269, - 0.0784146, - -0.003503646, - -0.031156048, - -0.033383664, - 0.01937351, - 0.037160154, - 0.030936565, - 0.1380185, - -0.002635351, - 0.0060887556, - 0.02401934, - 0.028815405, - 0.011467783, - 0.0028821875, - 0.009709699, - -0.010394833, - 0.025645163, - 0.044017233, - 0.008246027, - -0.023243824, - -0.025415665, - -0.022480464, - 0.016407011, - 0.0039797607, - -0.06682885, - -0.058947742, - -0.026093839, - 0.025729727, - -0.023972526, - -0.015408932, - -0.013048789, - -0.08130767, - 0.029917423, - -0.004828957, - -0.021634426, - 0.02587896, - 0.0044811512, - -0.013536556, - -0.014813144, - 0.046061084, - -0.00032990836, - 0.016869118, - -0.12181025, - 0.021046987, - -0.009420413, - -0.035062335, - 0.08011807, - 0.08462047, - 0.0019942592, - 0.05117461, - 0.05312125, - 0.0326721, - -0.04189356, - -0.055460256, - 0.01466244, - 0.03459353, - 0.095160365, - 0.0048437407, - 0.038064692, - 0.06401175, - 0.036864925, - 0.11731751, - 0.04750967, - 0.06286565, - -0.04375349, - 0.039547894, - -0.041467424, - 0.04528996, - -0.005650938, - 0.028340634, - 0.03510358, - -0.111572064, - 0.06749655, - 0.025440717, - -0.016470913, - -0.023179049, - 0.0256079, - -0.03489901, - -0.01430054, - -0.043748833, - -0.0060837497, - 0.044692438, - -0.0072164233, - -0.038763802, - -0.063516915, - -0.052999448, - 0.04417511, - 0.024537848, - -0.013432413, - -5.162713e-33, - -0.0114407325, - -0.003955193, - -0.04661703, - 0.0007875603, - -0.09029818, - -0.062948115, - 0.009710563, - 0.0001300855, - 0.04312154, - 0.022073459, - -0.04914153, - -0.08508943, - 0.11078909, - 0.017684652, - 0.056212854, - 0.03725169, - -0.114067726, - 0.011182615, - 0.0105617605, - 0.035042927, - -0.07395952, - 0.015640577, - -0.032705046, - -0.06488826, - -0.010690244, - -0.041422527, - -0.09704262, - -0.070222415, - 0.021709241, - 0.05029499, - -0.017807636, - 0.032029808, - -0.03788697, - 0.03136548, - 0.076138325, - -0.0007745447, - 0.034956176, - -0.06253955, - -0.006809682, - -0.026719663, - 0.11657212, - 0.050194807, - 0.06518189, - 0.06511228, - 0.025210718, - 0.03180309, - -0.059656575, - 0.08190252, - -0.028029623, - -0.04854541, - -0.034874525, - 0.030208217, - 0.035034187, - -0.018606044, - -0.038674414, - -0.048887372, - 0.021132758, - 0.08317319, - -0.0675027, - -0.05348525, - -0.080962874, - -0.06341586, - 0.011199907, - 0.0207855, - -0.06572508, - 0.022130286, - -0.10779961, - -0.011599286, - -0.051970255, - -0.15018743, - 0.11517539, - -0.03052435, - -0.0187536, - 0.052858524, - -0.06682251, - 0.04038606, - -0.028126227, - -0.016444748, - -0.02575468, - 0.06569969, - 0.08660793, - 0.010974402, - -0.0386448, - 0.027382996, - -0.06711591, - -0.0152584985, - -0.052659295, - 0.020921137, - 0.031379428, - 0.002811196, - 0.010419629, - 0.048391167, - 0.02201258, - 0.016886525, - -0.022732206, - -4.073636e-08, - -0.006513384, - 0.0014004525, - 0.053950045, - 0.015167113, - 0.018088367, - 0.08111558, - 0.0055300333, - -0.03707988, - -0.018737856, - 0.051793147, - -0.014813838, - -0.044947825, - -0.09278965, - -0.07260186, - -0.0041794567, - 0.14137252, - -0.022569552, - -0.030528586, - 0.047419064, - 0.066193394, - 0.07945365, - -0.023014557, - -0.053888306, - 0.104186185, - -0.08501846, - -0.03223301, - 0.05844058, - 0.0036631415, - -0.02911171, - -0.09349268, - -0.004123487, - -0.035597004, - 0.007244818, - 0.04035152, - 0.045556862, - 0.01838623, - 0.02155509, - -0.060004886, - -0.028096678, - 0.018191703, - -0.021443348, - -0.003914473, - 0.012175833, - -0.01663914, - 0.021617427, - 0.024437096, - -0.04434746, - -0.04760396, - -0.057683956, - -0.057734974, - -0.006014961, - -0.009957316, - -0.016418923, - -0.049850997, - 0.02046306, - 0.07664182, - -0.13724001, - 0.008844773, - -0.032054316, - 0.035961926, - -0.0063517457, - 0.11198241, - 0.1568678, - -0.0007999774 + -0.07448108, + 0.027982691, + -0.025962545, + 0.028414156, + -0.04874927, + -0.124489374, + -0.03775365, + 0.041172747, + -0.048783444, + -0.027774421, + -0.09272271, + 0.051921174, + 0.08087506, + 0.023085767, + 0.103185095, + -0.06142812, + -0.046623003, + 0.031264473, + -0.009095788, + -0.110987656, + -0.020735977, + 0.036462996, + -0.013348663, + 0.007442654, + 0.019446686, + 0.0043880027, + -0.0123794135, + -0.04474342, + -0.00010696763, + 0.027796188, + -0.05249273, + 0.062042117, + 0.019623421, + 0.022298045, + -0.01876838, + 0.06636658, + -0.036940884, + -0.09439301, + -0.04989112, + -0.016055813, + -0.08934105, + 0.07278765, + -0.073312856, + -0.027571253, + -0.06639977, + 0.015506035, + -0.004176694, + -0.032542672, + -0.035769954, + -0.026245229, + -0.09129098, + 0.022831371, + -0.05601971, + -0.103505865, + -0.023430603, + -0.01617043, + 0.060298156, + -0.011999374, + -0.00982143, + -0.15203232, + -0.07311755, + 0.022391053, + 0.08800625, + 0.062195398, + -0.04764835, + -0.05545306, + -0.036078423, + 0.017782934, + 0.08492913, + -0.050706394, + -0.09958507, + -0.029495796, + -0.002121337, + 0.08148674, + 0.030521393, + -0.12159759, + 0.04639748, + 0.0054555144, + -0.0076237656, + 0.04930283, + 0.001018987, + 0.01823945, + -0.056388717, + 0.09080432, + 0.03544767, + -0.062846325, + 0.05177355, + 0.07175976, + -0.045391884, + 0.009686718, + 0.030302709, + -0.058896482, + 0.03719664, + 0.004174063, + -0.014313601, + 0.06214871, + 0.026443055, + -0.054081496, + -0.04056011, + 0.010876058, + -0.0033277434, + -0.07736001, + 0.055489365, + 0.011366925, + 0.049955327, + 0.011093621, + 0.044155005, + -0.08873286, + 0.04789806, + -0.029256178, + -0.021238709, + -0.059048988, + -0.006010105, + -0.036286995, + 0.045776833, + 0.07393597, + -0.0043319017, + 0.07591234, + -0.0006300352, + 0.0063326987, + 0.019833053, + -0.008920521, + -0.0074224886, + -0.014964156, + 0.012450781, + 0.003317517, + -0.009942644, + 1.525195e-33, + -0.030182399, + -0.056817565, + -0.009954876, + 0.02231213, + 0.057156544, + -0.018560076, + 0.07843683, + -0.003509288, + -0.031122614, + -0.0333474, + 0.019342642, + 0.03716782, + 0.030942772, + 0.13801146, + -0.0026788223, + 0.0060844175, + 0.024037478, + 0.028806396, + 0.0114514725, + 0.0028755309, + 0.009741409, + -0.010365574, + 0.025636459, + 0.04402703, + 0.00824972, + -0.023288164, + -0.025415357, + -0.02247272, + 0.016395057, + 0.0039686435, + -0.06683203, + -0.058984432, + -0.026139224, + 0.02571613, + -0.023981044, + -0.01542635, + -0.013025425, + -0.08132036, + 0.029904919, + -0.0048653325, + -0.02163821, + 0.025880665, + 0.004492511, + -0.013551861, + -0.014834658, + 0.046109095, + -0.00031146017, + 0.016851023, + -0.12182429, + 0.021024965, + -0.009434213, + -0.03510208, + 0.080137864, + 0.08463277, + 0.0019426581, + 0.051176246, + 0.05314091, + 0.032667853, + -0.041880205, + -0.05545038, + 0.014655727, + 0.034564327, + 0.09517278, + 0.0048721586, + 0.038064517, + 0.064016655, + 0.036886543, + 0.11732628, + 0.04750395, + 0.062849574, + -0.043793496, + 0.039535545, + -0.0414883, + 0.045276705, + -0.005626682, + 0.028326502, + 0.03510831, + -0.11158364, + 0.067508236, + 0.025473768, + -0.016454473, + -0.023138152, + 0.02560681, + -0.03489655, + -0.0143142305, + -0.043763783, + -0.006103266, + 0.044694975, + -0.007177529, + -0.038755096, + -0.06350946, + -0.05295245, + 0.044151388, + 0.024555689, + -0.01345332, + -5.1627547e-33, + -0.011461753, + -0.003969141, + -0.04658726, + 0.0008026091, + -0.090269305, + -0.0629358, + 0.009687034, + 0.00015354449, + 0.043152034, + 0.022057066, + -0.049155302, + -0.08511033, + 0.110782035, + 0.017681966, + 0.056186423, + 0.03724774, + -0.114085265, + 0.011197734, + 0.010572792, + 0.03503156, + -0.07397689, + 0.0156148635, + -0.032688703, + -0.06490581, + -0.010675779, + -0.041401856, + -0.097037986, + -0.07025277, + 0.021750104, + 0.05030694, + -0.017832309, + 0.032031614, + -0.03788665, + 0.03141082, + 0.07613352, + -0.0007763451, + 0.034961626, + -0.06256205, + -0.006801991, + -0.026741587, + 0.11656076, + 0.05023973, + 0.06515106, + 0.06511257, + 0.025219081, + 0.03180813, + -0.05966658, + 0.08190675, + -0.028054262, + -0.048548922, + -0.03486897, + 0.03020514, + 0.035033725, + -0.018610824, + -0.038684692, + -0.048875436, + 0.021133669, + 0.08319505, + -0.06746284, + -0.053462982, + -0.08098418, + -0.06340421, + 0.011191566, + 0.020785637, + -0.06575731, + 0.02211741, + -0.10775702, + -0.011597437, + -0.051947355, + -0.1501959, + 0.11516611, + -0.030521782, + -0.018723903, + 0.052845538, + -0.06679985, + 0.040416736, + -0.028146135, + -0.01644884, + -0.025731068, + 0.06570538, + 0.0866128, + 0.010937938, + -0.03865133, + 0.027389226, + -0.06712724, + -0.015267271, + -0.05265448, + 0.020899015, + 0.031420153, + 0.002802588, + 0.010436373, + 0.048363067, + 0.021981295, + 0.01690293, + -0.022728851, + -4.0744272e-08, + -0.0065167644, + 0.0014059767, + 0.05391456, + 0.015178632, + 0.018086514, + 0.08112959, + 0.005525823, + -0.037069544, + -0.01871401, + 0.051793523, + -0.014797383, + -0.044994324, + -0.09279006, + -0.07259356, + -0.004214306, + 0.14136177, + -0.022566888, + -0.030480398, + 0.047431417, + 0.06623071, + 0.07947818, + -0.023033215, + -0.05389834, + 0.10418305, + -0.08498801, + -0.032223985, + 0.058419, + 0.0036608635, + -0.02912376, + -0.09348434, + -0.004131768, + -0.035598896, + 0.007222825, + 0.040373847, + 0.04553802, + 0.018402338, + 0.021517321, + -0.06000489, + -0.028075347, + 0.018188315, + -0.021463133, + -0.003939297, + 0.012185079, + -0.016664179, + 0.021595497, + 0.02443412, + -0.044382285, + -0.047587246, + -0.057701204, + -0.057771184, + -0.0060019926, + -0.0099875815, + -0.016420204, + -0.049889106, + 0.020464808, + 0.076619074, + -0.13720629, + 0.00883673, + -0.032044746, + 0.035911836, + -0.006365476, + 0.11197782, + 0.15684035, + -0.00079191517 ], [ - -0.0012985186, - 0.013428601, - 0.036027383, - 0.046960995, - -0.008376715, - -0.012273062, - 0.017215235, - -0.014517273, - -0.06755925, - 0.013262504, - -0.071071416, - 0.022008605, - 0.04802556, - -0.06656689, - -0.030001678, - 0.014703167, - 0.04293022, - 0.031151697, - -0.06519839, - -0.07397044, - 0.017323893, - -0.015189615, - -0.052739624, - 0.06344194, - 0.005378495, - 0.026332699, - 0.036680676, - 0.048806872, - -0.0044219326, - 0.010361781, - -0.008937124, - -0.07216964, - 0.050818473, - 0.017360602, - -0.061186902, - -0.010224321, - -0.06590306, - 0.06985154, - 0.028388679, - -0.037106816, - -0.052078426, - -0.07370584, - 0.023386989, - -0.025320385, - 0.06171919, - 0.11583571, - -0.08312255, - -0.08873915, - -0.04554808, - 0.021797463, - -0.12322211, - -0.02355109, - -0.0015547865, - -0.013524721, - -0.056223456, - 0.08805911, - 0.0332561, - 0.05172255, - 0.007508375, - -0.025260713, - 0.023153193, - -0.15516914, - -0.01075054, - 0.016285403, - 0.03417789, - 0.02007978, - -0.022245353, - 0.0509647, - -0.0054105176, - -0.040100772, - -0.020286275, - 0.10114523, - 0.0030004813, - 0.06618223, - 0.040104922, - -0.020045916, - -0.05968854, - -0.06369228, - 0.08476288, - 0.023561234, - -0.017190726, - -0.0057785655, - -0.02643019, - 0.09284292, - 0.048416004, - -0.068727545, - -0.02216159, - -0.015408143, - -0.011069366, - -0.017663702, - 0.025346316, - -0.03933665, - -0.0013904214, - -0.04090857, - -0.032088112, - 0.041472837, - 0.008925901, - -0.13771445, - 0.030238513, - 0.058210976, - 0.010595619, - 0.0924281, - -0.035886403, - -0.003660082, - 0.056327023, - -0.0040123863, - 0.035575725, - 0.014680677, - 0.10619057, - -0.13590562, - -0.05811401, - 0.04527551, - -0.06981517, - -0.049992837, - -0.041055493, - 0.012480766, - -0.04090579, - 0.02896762, - -0.022247234, - 0.087224506, - -0.009555419, - -0.025493871, - 0.0113851605, - 0.03364401, - 0.02181673, - -0.086783744, - -0.070866294, - 1.7912747e-33, - -0.04119901, - -0.100177474, - 0.006347325, - 0.0037340575, - 0.029203828, - -0.087258354, - -0.04274845, - -0.065680355, - 0.01794751, - 0.022274023, - -0.03245305, - -0.023515053, - 0.021447303, - 0.0950956, - 0.083449624, - 0.0428005, - 0.003910466, - 0.037924897, - 0.020041984, - -0.033424165, - 0.10205846, - -0.014394057, - 0.021688785, - -0.021577379, - -0.0074261655, - 0.04609739, - 0.06662811, - 0.06431144, - -0.010944364, - 0.016165929, - 0.030921511, - 0.017438315, - -0.07628473, - 0.027964544, - 0.05316952, - -0.06166001, - 0.00710056, - 0.0057538245, - 0.05521142, - 0.06931237, - -0.027706858, - -0.045808528, - 0.094666, - -0.02986965, - -0.04502887, - 0.017208695, - 0.016125973, - -0.0628507, - -0.045059443, - -0.045112878, - -0.005296992, - -0.019326933, - -0.045822155, - -0.02639405, - 0.01242909, - 0.08570191, - -0.003465873, - 0.003503288, - -0.012003436, - 0.006605807, - 0.03363934, - -0.001257058, - 0.04224235, - 0.055937544, - 0.017936032, - 0.07066278, - 0.1045465, - 0.062303454, - 0.044585444, - -0.002807214, - 0.02564102, - -0.002128406, - 0.040478833, - -0.01224923, - 0.052337434, - -0.0016797099, - 0.053469352, - -0.0077856537, - -0.028244767, - 0.018288352, - -0.046363432, - -0.04332065, - -0.026436778, - -0.11104876, - 0.008586205, - 0.0055207564, - -0.034841597, - -0.056425076, - -0.030253613, - 0.005325803, - -0.090041295, - -0.031432882, - -0.062356126, - 0.09982324, - -0.032827362, - -3.3549678e-33, - -0.027284035, - 0.010559345, - -0.021984268, - 0.12661384, - 0.0315912, - 0.033252638, - -0.051472977, - -0.030958762, - -0.04658957, - -0.0022805957, - -0.056222532, - 0.00796958, - 0.06494811, - 0.038894437, - -0.06838922, - 0.077499114, - -0.06790046, - 0.0064532245, - -0.040768467, - 0.037424307, - -0.072336495, - 0.06332956, - 0.014400053, - -0.05869224, - 0.031022472, - -0.019536898, - -0.07451289, - 0.03739678, - -0.02625108, - 0.02074715, - -0.031048505, - 0.0059261005, - -0.04759007, - -0.010896379, - 0.035239074, - 0.054979034, - 0.07011226, - -0.056623362, - -0.017411917, - 0.07528956, - 0.05387218, - 0.0028673257, - -0.07281712, - -0.07544035, - -0.012932695, - 0.011416252, - -0.08563262, - -0.0015282914, - 0.036346182, - -0.062029377, - -0.0050238175, - 0.02387278, - -0.008091779, - -0.035949487, - -0.034255754, - 0.0003292639, - -0.057821356, - 0.021184877, - 0.056231596, - 0.102305636, - -0.076927446, - -0.09633249, - 0.029132774, - 0.0010131018, - -0.010232655, - 0.055211753, - -0.021346482, - 0.048036017, - -0.008985098, - 0.0043310625, - 0.002983946, - 0.049164876, - -0.049816035, - 0.07115217, - -0.04826019, - -0.07298708, - -0.026493097, - -0.064357154, - -0.034591526, - -0.006029352, - 0.018753871, - -0.077848874, - -0.0046812696, - 0.04576945, - -0.043886483, - 0.012162078, - 0.02418125, - 0.035210256, - 0.0063425824, - -0.08672974, - -0.014485961, - 0.0486449, - -0.06944658, - 0.047546502, - -0.09639138, - -3.8882344e-08, - 0.020005174, - -0.0060803695, - 0.10673199, - -0.0072566518, - 0.11126952, - 0.07668037, - -0.0897575, - 0.109880716, - -0.060538035, - -0.061037064, - 0.046886686, - -0.016372517, - 0.01658076, - -0.012367154, - 0.0035005491, - 0.031382836, - 0.01833628, - 0.038030002, - -0.00055114034, - 0.019830866, - 0.11086577, - -0.02309543, - 0.04928018, - -0.049268693, - 0.037694186, - -0.10212397, - 0.021300899, - 0.004854364, - -0.026668059, - 0.04163984, - -0.037908267, - 0.029162008, - 0.03740134, - -0.015686596, - 0.09598688, - 0.14345205, - 0.04990253, - -0.11276881, - -0.062654205, - -0.038547758, - -0.030726157, - 0.08556472, - -0.048128515, - 0.04011241, - 0.014323266, - -0.021255655, - 0.048724912, - -0.057747725, - 0.009945408, - 0.0028096687, - 0.07980508, - 0.017901363, - 0.02239066, - 0.08985929, - 0.0665591, - 0.022021096, - 0.059401497, - -0.061183818, - 0.015351812, - 0.08374175, - -0.0016842537, - 0.08864498, - -0.027638372, - -0.06043769 + -0.0012923438, + 0.013419649, + 0.03603258, + 0.046982195, + -0.008386184, + -0.012245008, + 0.017257063, + -0.014495833, + -0.06755615, + 0.013220825, + -0.071046636, + 0.022029007, + 0.04805814, + -0.06659013, + -0.030023778, + 0.014715108, + 0.04294596, + 0.031195298, + -0.06522679, + -0.07396746, + 0.017329818, + -0.0151756415, + -0.052758723, + 0.06344977, + 0.005364444, + 0.02631366, + 0.03665044, + 0.048812985, + -0.0044375616, + 0.0103826355, + -0.0089511005, + -0.07216287, + 0.05088121, + 0.017377803, + -0.061182447, + -0.010244597, + -0.06587784, + 0.069840916, + 0.028359821, + -0.037131228, + -0.052071016, + -0.07370394, + 0.0233667, + -0.02532014, + 0.06171828, + 0.11584273, + -0.08307468, + -0.08872316, + -0.04554565, + 0.02177065, + -0.12324151, + -0.023568366, + -0.0015541487, + -0.013532973, + -0.056209136, + 0.0880576, + 0.03321554, + 0.05171784, + 0.0074756956, + -0.025275769, + 0.023162214, + -0.15517598, + -0.010777206, + 0.016303454, + 0.034188252, + 0.020134093, + -0.022240352, + 0.050957076, + -0.005396301, + -0.04007687, + -0.020301744, + 0.10113998, + 0.002977471, + 0.06617704, + 0.040134214, + -0.02005319, + -0.059682623, + -0.06369068, + 0.08473604, + 0.023557685, + -0.017191878, + -0.005820709, + -0.026404407, + 0.09280466, + 0.04844145, + -0.06875489, + -0.022161635, + -0.015402431, + -0.0111024445, + -0.017707076, + 0.025355583, + -0.039296508, + -0.001362202, + -0.040884525, + -0.03204941, + 0.04150212, + 0.008948646, + -0.13776794, + 0.030302526, + 0.058231197, + 0.010572606, + 0.09247389, + -0.035872795, + -0.0036602807, + 0.056347203, + -0.003996722, + 0.035537403, + 0.014696888, + 0.10615937, + -0.13590123, + -0.05810754, + 0.04527657, + -0.06982519, + -0.049982276, + -0.041045085, + 0.01247287, + -0.040934183, + 0.028955987, + -0.02226216, + 0.08722953, + -0.009548719, + -0.025511682, + 0.0114325285, + 0.03363939, + 0.021809513, + -0.08675585, + -0.07089411, + 1.7909231e-33, + -0.04121751, + -0.1001688, + 0.006345352, + 0.0037210584, + 0.029166285, + -0.0872215, + -0.04271259, + -0.06566409, + 0.017946582, + 0.022238955, + -0.03249184, + -0.02349789, + 0.021466883, + 0.09511927, + 0.08346572, + 0.042806614, + 0.0038908664, + 0.037915263, + 0.020043708, + -0.033399176, + 0.10208849, + -0.014397545, + 0.021684645, + -0.021582458, + -0.0074115414, + 0.046073515, + 0.06664795, + 0.06434497, + -0.010910654, + 0.016172478, + 0.030913299, + 0.017434347, + -0.0762684, + 0.027927354, + 0.053165767, + -0.061656844, + 0.007082498, + 0.0057526245, + 0.055203717, + 0.069314696, + -0.027693065, + -0.045786254, + 0.094618365, + -0.02984729, + -0.045069296, + 0.01723317, + 0.016129777, + -0.06281533, + -0.045081936, + -0.045089465, + -0.0053253355, + -0.019320533, + -0.045810748, + -0.02639149, + 0.012412514, + 0.08566385, + -0.0034776065, + 0.0035142878, + -0.012017715, + 0.006649936, + 0.033606175, + -0.0012646043, + 0.042252455, + 0.055928096, + 0.017948387, + 0.07064788, + 0.10451079, + 0.062350754, + 0.04458121, + -0.0028225682, + 0.02566386, + -0.0021405003, + 0.040477417, + -0.012259745, + 0.052335545, + -0.0017080541, + 0.05346329, + -0.007733562, + -0.028276777, + 0.018282998, + -0.046343774, + -0.043290336, + -0.026471136, + -0.11104024, + 0.008576623, + 0.005548108, + -0.034847535, + -0.056416124, + -0.030293388, + 0.0053394907, + -0.09004081, + -0.03141982, + -0.062330373, + 0.09981983, + -0.032840475, + -3.3540373e-33, + -0.027300175, + 0.010525057, + -0.021980286, + 0.12664026, + 0.031588834, + 0.033247624, + -0.05148502, + -0.03101089, + -0.0465964, + -0.0022529345, + -0.056195565, + 0.007953736, + 0.064945616, + 0.03884713, + -0.06837888, + 0.077476665, + -0.06788635, + 0.0064428714, + -0.040736765, + 0.037416343, + -0.07232494, + 0.063321635, + 0.014398016, + -0.05871896, + 0.031005096, + -0.019561818, + -0.07452502, + 0.037396118, + -0.026255993, + 0.020780139, + -0.031075457, + 0.0058948854, + -0.047562398, + -0.010866235, + 0.0352409, + 0.0549852, + 0.07012556, + -0.056673322, + -0.017415406, + 0.07528239, + 0.05387259, + 0.0028653517, + -0.07284915, + -0.07543174, + -0.012900278, + 0.011457189, + -0.08563738, + -0.0015463261, + 0.036361244, + -0.062004283, + -0.0050084046, + 0.023846988, + -0.008083734, + -0.03593437, + -0.034260865, + 0.000298229, + -0.0578704, + 0.021156322, + 0.056237947, + 0.102285825, + -0.07694436, + -0.096381366, + 0.029115336, + 0.001019501, + -0.010235284, + 0.055199094, + -0.021333022, + 0.04801045, + -0.008948923, + 0.0043332377, + 0.002985581, + 0.049172573, + -0.049805593, + 0.07117998, + -0.04823976, + -0.072981454, + -0.026498413, + -0.06437876, + -0.0346269, + -0.0060303714, + 0.018713593, + -0.07784192, + -0.0046854415, + 0.04578587, + -0.043880597, + 0.012154217, + 0.024205454, + 0.0352363, + 0.0063410155, + -0.086736806, + -0.014489626, + 0.048670504, + -0.06944819, + 0.047556538, + -0.096405424, + -3.8881783e-08, + 0.020024363, + -0.0060733794, + 0.10675529, + -0.0072445725, + 0.11130468, + 0.0766799, + -0.089739904, + 0.10989663, + -0.060538583, + -0.061066266, + 0.046883732, + -0.016365182, + 0.016547771, + -0.012390388, + 0.0035057077, + 0.031388927, + 0.018324051, + 0.038030062, + -0.0005554988, + 0.019816065, + 0.110884875, + -0.023082083, + 0.049298774, + -0.049228016, + 0.03771876, + -0.10209589, + 0.021328293, + 0.0048561115, + -0.026669646, + 0.04161308, + -0.037887473, + 0.029118432, + 0.03738528, + -0.015714107, + 0.0959638, + 0.1434109, + 0.049922757, + -0.11274395, + -0.06264596, + -0.038560014, + -0.03071335, + 0.08555022, + -0.048136428, + 0.0401538, + 0.014374478, + -0.021280114, + 0.04872567, + -0.057720494, + 0.009963986, + 0.002822142, + 0.079809405, + 0.017903175, + 0.022365756, + 0.08987974, + 0.06651197, + 0.022014199, + 0.059419304, + -0.06117766, + 0.015350715, + 0.08376493, + -0.0017018274, + 0.08864588, + -0.027652979, + -0.060420066 ], [ - -0.019079557, - 0.0820648, - -0.031636775, - -0.037772615, - -0.013885996, - -0.1508895, - -0.054257914, - 0.01382107, - 0.022319643, - 0.025744708, - -0.019006949, - 0.01595819, - 0.046914633, - -0.00899574, - 0.042291548, - 0.015646506, - -0.08305796, - 0.018408896, - -0.016524782, - -0.033079498, - -0.02110188, - -0.0419632, - -0.10861823, - 0.019554872, - -0.021874238, - 0.14247465, - -0.0012422869, - -0.058081616, - 0.00540865, - -0.03999031, - 0.012399737, - -0.014456615, - 0.10413924, - 0.08677547, - -0.07393572, - 0.031389575, - 0.07748671, - -0.041946597, - -0.092635125, - 0.019878551, - -0.09585241, - 0.063563004, - 0.0034580587, - 0.038572513, - -0.022447942, - 0.049308285, - -0.02643344, - -0.049521465, - -0.013297457, - 0.012233744, - -0.11695251, - 0.045083124, - -0.029010503, - 0.025497276, - 0.042470127, - 0.0707831, - 0.07058064, - 0.0035199749, - -0.06013254, - 0.041935362, - 0.016181944, - -0.07186833, - 0.014542711, - -0.0062323804, - 0.030054, - 0.047468036, - 0.011281582, - 0.013848972, - 0.04363679, - -0.021843519, - 0.022379788, - 0.047847077, - -0.04025328, - 0.09494594, - 0.03154395, - 0.013367471, - -0.03980583, - -0.02183361, - 0.028191755, - -0.03431455, - 0.019671934, - 0.043623473, - -0.042967957, - 0.05416258, - 0.023089629, - -0.05675844, - 0.016767101, - -0.027033433, - -0.03967794, - 0.022832932, - 0.074487366, - -0.0140734995, - -0.008246596, - 0.008278476, - -0.024108624, - -0.020060774, - 0.024378806, - -0.025747048, - 0.103516266, - -0.016442155, - 0.05220777, - 0.043397434, - 0.02440455, - 0.015943957, - -0.050434876, - -0.11145781, - 0.052034505, - -0.017928654, - -0.037932526, - -0.06774673, - -0.016093384, - 0.052765142, - -0.088646345, - -0.085484, - -0.07681618, - -0.093297966, - -0.12641862, - 0.013837021, - -0.03048377, - 0.009924758, - 0.039679028, - -0.01936025, - -0.028867563, - 0.00871666, - 0.01787285, - -0.11724568, - -0.12129051, - 1.35681665e-33, - -0.035628006, - -0.02325887, - -0.017038958, - 6.923209e-05, - 0.0072679906, - -0.0295577, - 0.022121288, - -0.010553554, - -0.06914253, - 0.04274084, - -0.03442124, - 0.041121893, - 0.017030265, - 0.0381245, - 0.1375638, - -0.008848526, - -0.0022947441, - 0.08370864, - -0.024741588, - -0.028515331, - 0.096916184, - -0.02108659, - 0.060407557, - -0.04129938, - 0.07367577, - 0.01852983, - -0.019585919, - 0.00791101, - -0.012426415, - -0.0051718187, - -0.02018194, - -0.011494365, - 0.0027114314, - 0.036264967, - 0.12386286, - 0.029567113, - 0.026756234, - -0.065749444, - 0.02609893, - -0.06232083, - 0.036904484, - 0.03028667, - 0.03411426, - 0.03521002, - -0.06369096, - -0.016598077, - -0.02021809, - -0.007230074, - 0.0040345713, - -0.07773345, - 0.06900628, - 0.012128798, - 0.02410663, - 0.0771743, - 0.027342282, - 0.03522959, - -0.046029396, - -0.061365336, - -0.026628872, - 0.08244359, - -0.062566556, - 0.009933027, - 0.034682497, - 0.023791147, - -0.005842399, - 0.021625068, - -0.026427383, - -0.020991165, - 0.11373874, - 0.03665437, - -0.008091131, - 0.0026228908, - -0.03253574, - 0.013892951, - -0.018594475, - -0.0059351088, - -0.012646403, - -0.04972099, - -0.048871726, - 0.027652413, - -0.08134938, - 0.0126620745, - 0.045843933, - 0.013398319, - 0.0023260224, - -0.05067545, - 0.04169543, - -0.01574087, - -0.07133913, - -0.016233964, - -0.018855713, - -0.0039056542, - 0.03401857, - -0.0093123065, - 0.0057734908, - -4.560601e-33, - 0.023695195, - -0.024489691, - -0.008312362, - -0.00066975394, - -0.02158263, - 0.0125598665, - -0.025738584, - -0.103652894, - -0.04000462, - 0.012098888, - -0.015197609, - 0.02018357, - 0.045623176, - -0.07047928, - 0.034468062, - 0.056500535, - -0.014972724, - -0.08429199, - -0.04942398, - -0.038302135, - -0.055943407, - 0.044392228, - -0.0019404019, - -0.07631783, - 0.034751914, - -0.0424522, - -0.07319884, - -0.08912471, - 0.08396021, - 0.034198415, - -0.055730376, - -0.017105753, - -0.0023682339, - -0.019267518, - 0.034007754, - -0.0067198407, - 0.07068643, - -0.013686713, - 0.03535481, - -0.011829574, - -0.011924876, - 0.08163265, - 0.011458664, - -0.049093027, - 0.046278197, - 0.029842824, - -0.035928097, - 0.13096437, - -0.0722123, - -0.053622153, - 0.047652073, - -0.032896154, - 0.033168253, - -0.053275317, - 0.119145334, - -0.013329809, - -0.080296695, - 0.01806636, - 0.028828703, - 0.012575126, - -0.08250055, - -0.07993187, - 0.0365166, - 0.048019268, - -0.0459654, - -0.039913233, - -0.019308258, - -0.11114867, - -0.12229502, - -0.08222976, - 0.014503677, - 0.041564006, - -0.054101657, - 0.12031798, - -0.10518697, - -0.033531662, - -0.046120696, - 0.015669933, - 0.031650025, - 0.08953049, - 0.062307738, - 0.023478396, - 0.013392765, - 0.043648973, - 0.017074035, - 0.030888386, - 0.052875523, - -0.055972677, - 0.015790377, - -0.04368904, - -0.039097052, - -0.020597953, - -0.018675094, - 0.08349847, - -0.017391236, - -3.870914e-08, - -0.05217957, - -0.0943954, - 0.009313268, - -0.024596054, - 0.0457224, - 0.0017694158, - -0.0194238, - 0.14304265, - -0.00092139974, - -0.018642776, - 0.060916223, - -0.022210617, - -0.06669, - -0.042800087, - 0.076100215, - 0.05237621, - 0.08171605, - -0.13214897, - 0.015094836, - 0.075452864, - 0.01636198, - 0.0030703964, - -0.061852757, - 0.07880552, - 0.04179526, - -0.04381105, - 0.057303566, - 0.0139259575, - -0.015837422, - 0.0027170512, - -0.0029033618, - -0.02796994, - 0.035219938, - 0.07358342, - 0.115382664, - 0.008049736, - 0.054797564, - 0.070874535, - -0.04053772, - -0.07585998, - 0.015316053, - -0.014189948, - -0.038860295, - 0.029442793, - 0.061300512, - 0.025522308, - -0.039504033, - 0.11314281, - -0.028287454, - 0.031891253, - -0.038770907, - 0.029970054, - -0.020935897, - -0.004616352, - -0.06046541, - 0.010621891, - -0.0069159092, - -0.04626887, - 0.040723223, - 0.03980271, - -0.016016755, - 0.025667662, - 0.035244495, - -0.026702441 + -0.019089537, + 0.08206227, + -0.031629756, + -0.037748322, + -0.013907723, + -0.15086435, + -0.054227855, + 0.013812081, + 0.022318492, + 0.025760967, + -0.018970305, + 0.0159997, + 0.046886247, + -0.008989786, + 0.042260803, + 0.01563633, + -0.08306234, + 0.018418225, + -0.016524842, + -0.033054315, + -0.021094276, + -0.04198475, + -0.108629815, + 0.019558346, + -0.021839257, + 0.14248955, + -0.0012803682, + -0.058087774, + 0.005395786, + -0.040014874, + 0.012412929, + -0.014448109, + 0.10412988, + 0.08678136, + -0.07392144, + 0.031378184, + 0.077501394, + -0.04197698, + -0.092644565, + 0.019878637, + -0.09584833, + 0.06355258, + 0.0034316017, + 0.03860985, + -0.022438047, + 0.04932071, + -0.026379092, + -0.049524873, + -0.013308545, + 0.012192514, + -0.11695286, + 0.04510036, + -0.029017858, + 0.025516428, + 0.04245081, + 0.070753604, + 0.07057494, + 0.003524953, + -0.06010962, + 0.041959174, + 0.016197778, + -0.07186037, + 0.014555853, + -0.006213116, + 0.030063417, + 0.047432736, + 0.011306432, + 0.013843393, + 0.0436187, + -0.021850524, + 0.022346757, + 0.047835413, + -0.04025223, + 0.09492459, + 0.03155159, + 0.013348888, + -0.039819352, + -0.021837216, + 0.028181475, + -0.03434981, + 0.019666592, + 0.043579087, + -0.042940862, + 0.054164745, + 0.02308801, + -0.056740467, + 0.016757911, + -0.02701336, + -0.039681926, + 0.022773864, + 0.074453875, + -0.01407503, + -0.008249863, + 0.008273288, + -0.024091411, + -0.020071099, + 0.024399305, + -0.025779521, + 0.1035294, + -0.016452465, + 0.05220051, + 0.043400586, + 0.024392875, + 0.0160118, + -0.050395392, + -0.11149879, + 0.05203916, + -0.017942373, + -0.03793447, + -0.06775703, + -0.01611577, + 0.05274979, + -0.08863033, + -0.085470706, + -0.076794446, + -0.09332248, + -0.1264284, + 0.013839316, + -0.030490262, + 0.009920159, + 0.03968685, + -0.01939706, + -0.028892461, + 0.008741198, + 0.017886965, + -0.117217556, + -0.1212998, + 1.35733635e-33, + -0.035622492, + -0.023267707, + -0.017018162, + 0.00010073695, + 0.007257954, + -0.029587401, + 0.022087794, + -0.010561547, + -0.06912062, + 0.04277785, + -0.034413584, + 0.041110493, + 0.017055655, + 0.038174715, + 0.13757399, + -0.008806284, + -0.0023235404, + 0.08372674, + -0.024748268, + -0.028528849, + 0.096861266, + -0.02111509, + 0.06039901, + -0.041284908, + 0.07366366, + 0.018533891, + -0.019621244, + 0.00789655, + -0.012412154, + -0.005184189, + -0.0202234, + -0.011487718, + 0.0026882978, + 0.036282968, + 0.12384692, + 0.029563135, + 0.02673901, + -0.06578298, + 0.02610267, + -0.062275145, + 0.036926493, + 0.030272253, + 0.034105044, + 0.03516919, + -0.06365454, + -0.016557874, + -0.020214476, + -0.007219471, + 0.004009068, + -0.07774858, + 0.06894675, + 0.012156706, + 0.024095584, + 0.07716194, + 0.027376112, + 0.03524163, + -0.046042208, + -0.061379924, + -0.026633548, + 0.08248479, + -0.06261388, + 0.009910456, + 0.034668844, + 0.023772387, + -0.005869554, + 0.02162769, + -0.026385942, + -0.02100117, + 0.11375441, + 0.03666832, + -0.008121711, + 0.0026215075, + -0.032531988, + 0.01391055, + -0.018540533, + -0.0059300573, + -0.012669122, + -0.04971856, + -0.048864197, + 0.027610987, + -0.08137648, + 0.012624587, + 0.045806322, + 0.01336533, + 0.002328637, + -0.050664812, + 0.041695803, + -0.015773693, + -0.07136885, + -0.016258836, + -0.018871423, + -0.0038626953, + 0.03402061, + -0.009335479, + 0.005747506, + -4.5611018e-33, + 0.023689948, + -0.02445775, + -0.00834689, + -0.00063168275, + -0.021578811, + 0.012567475, + -0.025760869, + -0.10368349, + -0.03997725, + 0.01210385, + -0.015231519, + 0.02017564, + 0.045654193, + -0.07050829, + 0.034459736, + 0.056491707, + -0.014989821, + -0.08433123, + -0.049400527, + -0.03832157, + -0.055948768, + 0.044390477, + -0.001941214, + -0.0763155, + 0.034730915, + -0.04243297, + -0.07322386, + -0.08912488, + 0.083965875, + 0.034240186, + -0.055734336, + -0.017151177, + -0.0023456868, + -0.019274496, + 0.03401833, + -0.006712739, + 0.070724845, + -0.013663151, + 0.035358265, + -0.011840785, + -0.011920096, + 0.081632204, + 0.011438198, + -0.04905726, + 0.04624871, + 0.029794158, + -0.035954632, + 0.1309978, + -0.0722, + -0.053626865, + 0.047662914, + -0.032893717, + 0.03320312, + -0.053293463, + 0.11909418, + -0.013308413, + -0.08026765, + 0.018056376, + 0.028816566, + 0.012597203, + -0.082487956, + -0.07992265, + 0.03653938, + 0.048042614, + -0.04597376, + -0.039927375, + -0.019282784, + -0.11115308, + -0.12229221, + -0.08222088, + 0.014523922, + 0.041549023, + -0.054067343, + 0.12032739, + -0.10513437, + -0.03352011, + -0.046141136, + 0.015660388, + 0.03162219, + 0.089564346, + 0.06229127, + 0.02344754, + 0.013432015, + 0.04364802, + 0.017062847, + 0.030911682, + 0.052861545, + -0.05597565, + 0.015810143, + -0.04374839, + -0.039106574, + -0.020592151, + -0.01868341, + 0.08352379, + -0.017375095, + -3.8713683e-08, + -0.052152414, + -0.09442023, + 0.009305927, + -0.024598995, + 0.04574071, + 0.0017779457, + -0.019384999, + 0.14307584, + -0.00092140987, + -0.018639628, + 0.06094085, + -0.022180414, + -0.06670714, + -0.042788457, + 0.07614433, + 0.052368972, + 0.08171796, + -0.13214965, + 0.015069824, + 0.07545052, + 0.016364794, + 0.0030805927, + -0.06188439, + 0.07879054, + 0.04179921, + -0.043787137, + 0.05729686, + 0.013950966, + -0.01580636, + 0.002741003, + -0.002896178, + -0.027976623, + 0.0352471, + 0.07360851, + 0.11537727, + 0.008016604, + 0.054790642, + 0.070841216, + -0.040544577, + -0.07585315, + 0.015317468, + -0.014144724, + -0.03884744, + 0.029432015, + 0.061295677, + 0.025552604, + -0.03950773, + 0.1131327, + -0.028318027, + 0.031907115, + -0.038748857, + 0.029967804, + -0.020923622, + -0.0045868345, + -0.060423743, + 0.01062511, + -0.006921613, + -0.046255972, + 0.04074385, + 0.039824147, + -0.016014125, + 0.025676023, + 0.03524506, + -0.0267346 ], [ - -0.053175602, - -0.047849268, - 0.049600203, - -0.009332594, - -0.05626027, - -0.03703611, - 0.015297836, - 0.0033727393, - 0.044511985, - 0.016425023, - -0.06529153, - 0.046528336, - 0.012637323, - 0.025194079, - -0.1143288, - 0.027321098, - -0.052430134, - 0.060264964, - -0.046056643, - -0.022868538, - 0.016518874, - 0.014427887, - -0.077468514, - 0.01650613, - -0.067144066, - 0.120887764, - -0.0022775852, - -0.0005598929, - 0.031006373, - 0.031167403, - 0.10499404, - -0.069391765, - -0.01322822, - 0.028970728, - -0.08779589, - 0.05563035, - -0.091597155, - -0.018200668, - -0.024829883, - -0.020258859, - 0.0131373005, - -0.0007341065, - 0.0018953033, - 0.006834895, - 0.08603948, - 0.06189398, - -0.07733514, - -0.047121815, - -0.04994335, - -0.0089659095, - -0.0880838, - 0.0011172506, - -0.015044709, - -0.0075995945, - 0.085313074, - 0.059796136, - 0.02457739, - 0.0378336, - -0.051707182, - 0.031467274, - 0.113771856, - -0.044192057, - 0.0096846735, - 0.006033161, - 0.030144352, - 0.07118354, - -0.013839908, - 0.036214717, - 0.004951509, - -0.07481083, - 0.09734058, - 0.07162632, - -0.009135306, - -0.009563247, - 0.042295255, - 0.0117468545, - 0.03281954, - 0.018608347, - 0.012542441, - -0.009309551, - -0.034870803, - 0.016498035, - 0.0054994198, - 0.038178287, - 0.09602082, - -0.0020852594, - -0.020779438, - 0.01808113, - -0.03249026, - 0.012480446, - -0.014463354, - -0.06702938, - -0.09548575, - -0.103447035, - -0.0009932001, - -0.0030760013, - 0.026984407, - -0.033981565, - 0.0011538514, - -0.009027189, - -0.048636526, - 0.0029721952, - -0.041503906, - -0.03960792, - 0.07517321, - 0.031135045, - 0.030046917, - 0.033542294, - 0.11397492, - -0.082903914, - -0.109131016, - 0.03003371, - -0.041856304, - 0.04223555, - 0.033319004, - -0.03889455, - 0.020930232, - 0.02838724, - 0.0545114, - 0.09626628, - -0.0035141057, - -0.015085271, - -0.09259153, - -0.056270823, - -0.0033157181, - -0.029304419, - -0.114175975, - 1.50678135e-33, - -0.0453055, - -0.07348326, - 0.034691177, - -0.0672167, - 0.023145972, - -0.050515983, - -0.017413607, - -0.0058405283, - 0.052108254, - -0.017992783, - -0.10167575, - 0.016488168, - -0.0059505017, - 0.08831343, - 0.047385737, - -0.06261416, - -0.03727668, - -0.049049053, - 0.061813977, - -0.11765181, - 0.014997916, - -0.07084365, - 0.07316741, - -0.010097435, - -0.0045747026, - 0.0014380639, - 0.0123074865, - -0.018593263, - 0.019023519, - -0.0076754233, - -0.008543783, - 0.023825979, - -0.0074089407, - -0.042009465, - -0.008104463, - -0.008959146, - 0.11069426, - -0.028461525, - 0.0375111, - 0.047092855, - 0.062606744, - -0.049568158, - 0.06266772, - 0.0053055165, - 0.024054594, - 0.034305595, - -0.017003167, - -0.033732932, - 0.012580805, - -0.057429112, - -0.046275277, - -0.0003945471, - 0.02263768, - -0.10997523, - 0.09229477, - 0.048902728, - -0.044187002, - 0.05441158, - -0.0057972632, - 0.04834593, - 0.035639632, - -0.015485863, - -0.008143862, - 0.092880696, - 0.11231507, - 0.047900956, - -0.017541546, - -0.009539733, - 0.06213859, - -0.0040546083, - 0.003987384, - 0.09531304, - -0.056603517, - -0.058908645, - -0.013667576, - 0.009745052, - -0.047453303, - -0.06157018, - -0.08587985, - 0.05011287, - -0.02779263, - -0.008005466, - -0.068401575, - 0.032416083, - 0.015329646, - 0.08306027, - 0.06357283, - -0.00512495, - -0.01188288, - -0.051893827, - -0.008702526, - -0.031820606, - 0.043191314, - 0.00033676252, - -0.0012971128, - -2.3314325e-33, - -0.084871486, - 0.023456383, - -0.05555233, - 0.028799664, - 0.059832368, - 0.044252343, - -0.069759004, - -0.08750932, - -0.023541803, - 0.076747485, - 0.015193914, - 0.01961009, - -0.05837612, - 0.01878715, - 0.007621002, - -0.015989477, - -0.057301812, - -0.0426483, - 0.10103607, - -0.03979966, - -0.03179959, - 0.031775456, - -0.05796451, - -0.036753736, - 0.02731803, - -0.0069522746, - -0.07528311, - 0.049413346, - 0.012717442, - 0.10011093, - -0.03626197, - -0.0480568, - 0.029068258, - 0.017971879, - 0.04527712, - 0.10260452, - 0.0050376365, - -0.05527294, - 0.008323474, - -0.05968206, - 0.020133188, - 0.009408143, - -0.06650717, - -0.029911388, - 0.0434493, - -0.068347804, - -0.076517664, - 0.040012714, - -0.064759254, - 0.07230589, - 0.04662111, - -0.016778024, - -0.048703287, - -0.08456952, - -0.052551, - 0.03198548, - 0.024643922, - 0.02381256, - 0.07633642, - -0.040978454, - -0.033941545, - -0.11415368, - 0.067884214, - 0.009646611, - -0.06406483, - 0.02458555, - 0.024917984, - -0.0041125035, - 0.018718159, - -0.03810467, - 0.014550252, - 0.06850764, - 0.018693756, - 0.059391443, - 0.023741595, - -0.00974202, - -0.06651425, - 0.020927029, - -0.019371133, - 0.01486253, - 0.022714352, - -0.022630502, - 0.010553403, - 0.056958556, - 0.072571084, - 0.06506972, - -0.010076679, - 0.079096675, - 0.035260018, - -0.023826087, - 0.017108874, - 0.087825984, - 0.0059526036, - 0.0074271723, - -0.109360956, - -2.8789334e-08, - -0.05233612, - -0.087671354, - 0.066617705, - 0.013912193, - 0.099948354, - -0.02244002, - 0.062119395, - 0.027858257, - -0.064296365, - -0.038687464, - 0.025052465, - 0.008087938, - -0.024082167, - 0.011928929, - 0.0871567, - 0.012509529, - 0.064730704, - -0.027875392, - 0.039984196, - -0.012320989, - 0.023347521, - -0.032504674, - -0.042588573, - 0.107389025, - 0.037681337, - -0.06630358, - -0.056843463, - -0.0052555962, - -0.069520734, - 0.100924, - -0.033373408, - 0.02178169, - 0.017423104, - 0.01809016, - 0.02630718, - 0.066061474, - 0.059622575, - -0.065362565, - -0.11576683, - -0.071220115, - -0.023386031, - 0.042642016, - 0.043645483, - -0.036648206, - 0.05023266, - 0.0031018173, - 0.057091165, - -0.03462122, - 0.025469558, - -0.046201944, - -0.06719312, - 0.06058484, - -0.041243985, - -0.019823411, - -0.013743429, - -0.061215486, - 0.014752095, - -0.07632035, - -0.056729525, - 0.050518394, - -0.0360576, - 0.12239626, - 0.06431157, - -0.038293842 + -0.053171553, + -0.047855794, + 0.04959839, + -0.009352584, + -0.056259144, + -0.036997948, + 0.01525368, + 0.0033788579, + 0.04453428, + 0.016438372, + -0.065293424, + 0.04655176, + 0.012637792, + 0.025149647, + -0.11436081, + 0.027283441, + -0.052422393, + 0.060236752, + -0.046064522, + -0.022863738, + 0.016536511, + 0.014447978, + -0.07744467, + 0.016475804, + -0.067145765, + 0.120901324, + -0.0022643541, + -0.0005619333, + 0.03098974, + 0.03116176, + 0.10501578, + -0.06940328, + -0.013246061, + 0.029016647, + -0.08779694, + 0.055636257, + -0.09158273, + -0.018188708, + -0.024831342, + -0.020263424, + 0.013102336, + -0.0007477728, + 0.0018712403, + 0.0068353964, + 0.08601601, + 0.061896168, + -0.07733195, + -0.047134392, + -0.04994557, + -0.008955441, + -0.08808325, + 0.0011078792, + -0.015078675, + -0.007628681, + 0.08530312, + 0.059783977, + 0.024557464, + 0.037825108, + -0.05171798, + 0.03148071, + 0.11377193, + -0.04417297, + 0.009659848, + 0.0060449084, + 0.030134702, + 0.07118153, + -0.013864897, + 0.03624278, + 0.0049465275, + -0.07480586, + 0.09733932, + 0.071613275, + -0.009146446, + -0.009571701, + 0.042258315, + 0.011740325, + 0.032803785, + 0.018631615, + 0.012556345, + -0.009346388, + -0.03489368, + 0.01649207, + 0.005488214, + 0.03819102, + 0.09597803, + -0.002047146, + -0.020768773, + 0.018077927, + -0.032444023, + 0.012474241, + -0.014445184, + -0.0670006, + -0.095488854, + -0.10345397, + -0.0009862595, + -0.0030658073, + 0.027003448, + -0.033961065, + 0.0011482734, + -0.009025799, + -0.048620287, + 0.0029769312, + -0.04154341, + -0.0395945, + 0.07520094, + 0.031153427, + 0.030031031, + 0.03353441, + 0.11403943, + -0.082912125, + -0.109138384, + 0.030059446, + -0.041853014, + 0.042241115, + 0.033335667, + -0.038876496, + 0.02092849, + 0.028346559, + 0.054482125, + 0.09627962, + -0.0035115955, + -0.015083763, + -0.092599295, + -0.056257337, + -0.00332258, + -0.02934002, + -0.11417531, + 1.5075675e-33, + -0.04527847, + -0.07345357, + 0.034714583, + -0.067186035, + 0.023143126, + -0.05054431, + -0.017398916, + -0.0058387746, + 0.052131217, + -0.017985696, + -0.10168014, + 0.016505243, + -0.005961273, + 0.08834502, + 0.047341425, + -0.06262999, + -0.03724901, + -0.0490674, + 0.061806694, + -0.117662214, + 0.014966754, + -0.07085228, + 0.07317225, + -0.010064827, + -0.004601465, + 0.0014379362, + 0.0122654615, + -0.018565418, + 0.018996973, + -0.0076706754, + -0.0085447915, + 0.023833418, + -0.0074106916, + -0.04202295, + -0.008097604, + -0.0089935325, + 0.11068735, + -0.028457392, + 0.037548065, + 0.04710371, + 0.062597714, + -0.049594503, + 0.06267496, + 0.005339454, + 0.024064569, + 0.034303125, + -0.016984673, + -0.03375307, + 0.012577206, + -0.05741818, + -0.046267692, + -0.00036155691, + 0.02268587, + -0.109952465, + 0.09230675, + 0.048918508, + -0.044157643, + 0.05441931, + -0.0058244704, + 0.04833069, + 0.035635386, + -0.015495411, + -0.008146981, + 0.092891365, + 0.112310715, + 0.047900427, + -0.017513819, + -0.009520781, + 0.06212363, + -0.0040008924, + 0.00397841, + 0.09532846, + -0.05659656, + -0.058885954, + -0.013697212, + 0.009742546, + -0.04745855, + -0.061571207, + -0.085869245, + 0.05009574, + -0.027810305, + -0.007983068, + -0.06844095, + 0.032406274, + 0.015316275, + 0.0830624, + 0.063605405, + -0.005157704, + -0.011889667, + -0.05187598, + -0.0087124705, + -0.031850815, + 0.043204896, + 0.00032051498, + -0.0012597291, + -2.3328516e-33, + -0.08486178, + 0.023463517, + -0.05558325, + 0.028823433, + 0.0598007, + 0.044241305, + -0.06976774, + -0.08749109, + -0.023545535, + 0.0767821, + 0.015185076, + 0.019631226, + -0.058358442, + 0.018799065, + 0.0076146126, + -0.015977694, + -0.057259887, + -0.042667117, + 0.101026215, + -0.03983678, + -0.03180352, + 0.03177619, + -0.057957705, + -0.036778692, + 0.027305948, + -0.0069477605, + -0.0753, + 0.049428534, + 0.012732314, + 0.10010171, + -0.036260307, + -0.048061043, + 0.029081684, + 0.01795974, + 0.045303203, + 0.102590606, + 0.005036657, + -0.05526093, + 0.008327211, + -0.05970527, + 0.020131486, + 0.009408121, + -0.06648779, + -0.029893365, + 0.0434368, + -0.0683305, + -0.07649664, + 0.039999247, + -0.06477932, + 0.07227491, + 0.046653986, + -0.016773192, + -0.048649658, + -0.08454509, + -0.05255037, + 0.0319589, + 0.024662357, + 0.023793997, + 0.076360136, + -0.040995322, + -0.033935655, + -0.11416756, + 0.06787201, + 0.009610846, + -0.064101316, + 0.024561828, + 0.024906442, + -0.0041048713, + 0.018717252, + -0.038110614, + 0.0145301875, + 0.068478055, + 0.018691448, + 0.05943308, + 0.023695862, + -0.009747667, + -0.066519946, + 0.0209059, + -0.019389415, + 0.014860701, + 0.022718104, + -0.022605024, + 0.0105253365, + 0.05693715, + 0.07257885, + 0.06504599, + -0.010055237, + 0.07908256, + 0.035240322, + -0.02378674, + 0.017134566, + 0.0878081, + 0.005987074, + 0.007431842, + -0.10935983, + -2.8794002e-08, + -0.05234688, + -0.08765063, + 0.06662866, + 0.013907749, + 0.0999487, + -0.022422735, + 0.06214868, + 0.027856557, + -0.06424995, + -0.038701627, + 0.025059296, + 0.00807731, + -0.024077412, + 0.011949065, + 0.08715261, + 0.012486595, + 0.06470489, + -0.027933354, + 0.039985545, + -0.012295149, + 0.02333007, + -0.03250732, + -0.04260915, + 0.10736886, + 0.037696708, + -0.06628188, + -0.056817852, + -0.005238912, + -0.069547325, + 0.100934796, + -0.033363372, + 0.021774344, + 0.017414633, + 0.018075803, + 0.026276791, + 0.066073745, + 0.059642654, + -0.065390244, + -0.115749314, + -0.07125786, + -0.023382567, + 0.042660285, + 0.043636538, + -0.03665277, + 0.050204884, + 0.0030947176, + 0.057122562, + -0.034636553, + 0.025459053, + -0.046185397, + -0.067215376, + 0.06057241, + -0.041255984, + -0.019857686, + -0.013778329, + -0.06125949, + 0.014752149, + -0.07630465, + -0.056748062, + 0.0505062, + -0.036068004, + 0.12241577, + 0.06429002, + -0.038303368 ] ] } diff --git a/tests/integration/recordings/responses/d0ac68cbde69.json b/tests/integration/recordings/responses/d0ac68cbde69.json index eade8f14e..e5b90618e 100644 --- a/tests/integration/recordings/responses/d0ac68cbde69.json +++ b/tests/integration/recordings/responses/d0ac68cbde69.json @@ -16,9 +16,9 @@ "model": "llama3.2:3b-instruct-fp16", "name": "llama3.2:3b-instruct-fp16", "digest": "195a8c01d91ec3cb1e0aad4624a51f2602c51fa7d96110f8ab5a20c84081804d", - "expires_at": "2025-08-04T16:00:57.955349-07:00", - "size": 8581748736, - "size_vram": 8581748736, + "expires_at": "2025-08-05T14:12:18.480323-07:00", + "size": 7919570944, + "size_vram": 7919570944, "details": { "parent_model": "", "format": "gguf", @@ -29,6 +29,24 @@ "parameter_size": "3.2B", "quantization_level": "F16" } + }, + { + "model": "all-minilm:l6-v2", + "name": "all-minilm:l6-v2", + "digest": "1b226e2802dbb772b5fc32a58f103ca1804ef7501331012de126ab22f67475ef", + "expires_at": "2025-08-05T14:10:20.883978-07:00", + "size": 590204928, + "size_vram": 590204928, + "details": { + "parent_model": "", + "format": "gguf", + "family": "bert", + "families": [ + "bert" + ], + "parameter_size": "23M", + "quantization_level": "F16" + } } ] } diff --git a/tests/integration/recordings/responses/da531c71e64f.json b/tests/integration/recordings/responses/da531c71e64f.json new file mode 100644 index 000000000..4c77f5fc0 --- /dev/null +++ b/tests/integration/recordings/responses/da531c71e64f.json @@ -0,0 +1,421 @@ +{ + "request": { + "method": "POST", + "url": "http://localhost:11434/api/embeddings", + "headers": {}, + "body": { + "model": "all-minilm:l6-v2", + "input": [ + "Python programming language" + ] + }, + "endpoint": "/api/embeddings", + "model": "all-minilm:l6-v2" + }, + "response": { + "body": { + "__type__": "ollama._types.EmbedResponse", + "__data__": { + "model": "all-minilm:l6-v2", + "created_at": null, + "done": null, + "done_reason": null, + "total_duration": 105895041, + "load_duration": 91634666, + "prompt_eval_count": 3, + "prompt_eval_duration": null, + "eval_count": null, + "eval_duration": null, + "embeddings": [ + [ + -0.063880146, + 0.013411989, + -0.054502595, + 0.01193493, + -0.074262686, + -0.13344447, + 0.04294062, + 0.045387108, + -0.06949706, + -0.035939943, + 0.01200873, + 0.0068830596, + 0.08886977, + 0.0026030506, + 0.032482542, + -0.007821568, + -0.05044649, + 0.006662123, + 0.027794942, + -0.12791364, + 0.00062353734, + 0.045270294, + -0.03605076, + 0.044243146, + 0.0129354475, + -0.0092799105, + 0.011904844, + 0.026060482, + 0.020055141, + -0.03368774, + -0.028043076, + 0.087557025, + 0.059002083, + 0.053893365, + 0.02027196, + 0.06840361, + -0.03180594, + -0.087597735, + -0.11277839, + 0.022651086, + -0.09037903, + -0.0033202847, + -0.040132593, + -0.034084503, + -0.032953303, + 0.02925268, + -0.03903928, + 0.04551951, + -0.0331016, + -0.006518362, + -0.09629851, + -0.011739161, + -0.052575007, + -0.064773224, + 0.031043475, + -0.012586444, + 0.09737276, + 0.005224713, + -0.035071153, + -0.1404299, + -0.06678175, + 0.03654573, + -0.039277818, + 0.07014256, + -0.0010227569, + -0.026846789, + -0.0175696, + 0.03044068, + 0.06403526, + -0.031643596, + -0.14598879, + -0.045400888, + -0.018469285, + 0.06689445, + 0.030553635, + -0.12255281, + 0.061046645, + -0.05678168, + -0.005118667, + -0.0087622, + 0.006514719, + -0.016424034, + -0.033650044, + 0.08491301, + -0.00029260007, + -0.07339515, + 0.038627055, + 0.15695965, + 0.010035773, + 0.025318887, + -0.0021428047, + -0.04613549, + 0.06244243, + -0.019905778, + -0.05471386, + 0.09796629, + 0.0384793, + -0.072424814, + -0.038704097, + 0.07158691, + 0.007360897, + -0.05120446, + 0.0313513, + -0.032230332, + 0.039326303, + -0.009643992, + 0.069905065, + -0.052026685, + 0.049440835, + -0.04272916, + -0.0037707465, + -0.04155246, + -0.0561972, + -0.03340213, + 0.05105359, + 0.038616214, + -0.0029470131, + 0.08188407, + -0.0035886324, + 0.04530431, + 0.0068888925, + 0.016499842, + 0.016347302, + 0.007283021, + -0.021663606, + -0.0046215886, + -0.007931065, + -4.1536508e-33, + -0.045777988, + -0.050903402, + -0.038634304, + 0.0100991195, + 0.070007294, + -0.025182785, + 0.1050647, + -0.0049731904, + -0.064141616, + -0.047639705, + 0.012718577, + 0.05198462, + -0.016051587, + 0.08170543, + 0.024008816, + -0.020879291, + 0.045706064, + 0.091577366, + 0.02512945, + 0.019055998, + 0.048144504, + 0.097951256, + 0.034154113, + 0.03543114, + 0.011410896, + -0.043446988, + -0.0041784984, + -0.05564714, + 0.01147717, + 0.0071039577, + -0.06426582, + -0.020623188, + -0.0045247558, + -0.012943628, + 0.02658834, + -0.012385487, + 0.008399212, + -0.06824828, + 0.04683057, + -0.04165085, + -0.025662417, + -0.0038799767, + 0.05007075, + -0.008117481, + -0.023308154, + 0.023914568, + 0.0015741173, + 0.046142872, + -0.06898886, + 0.041611847, + 0.0045286645, + -0.047628563, + 0.054236773, + 0.06972688, + -0.016889753, + 0.04806098, + 0.012714234, + 0.0022186628, + -0.006355918, + -0.031550523, + 0.023726372, + 0.06859327, + 0.077228814, + -0.01227583, + 0.03901903, + 0.034360897, + 0.03032876, + 0.058690928, + 0.08030179, + 0.06976231, + -0.09047136, + 0.02376998, + -0.008751518, + 0.038334776, + -0.02751323, + 0.023137644, + 0.027101006, + -0.08135271, + -0.010334998, + 0.04730408, + -0.02033998, + -0.026008504, + -0.017415512, + -0.0035714875, + -0.018727385, + -0.037389226, + 0.041064497, + 0.05317889, + -0.0055602547, + -0.058561854, + -0.072036326, + -0.075019896, + 0.04825644, + 0.011348427, + -0.02259257, + 1.3515749e-33, + 0.006240622, + 0.031606406, + -0.036119435, + -0.0016494404, + -0.08255665, + -0.06069396, + 0.059934463, + 0.014492232, + 0.059514895, + 0.027053975, + -0.011601325, + -0.057609312, + 0.10365583, + -0.002784741, + 0.07693759, + 0.019432511, + -0.052210074, + 0.015158053, + -0.0012768542, + 0.027789148, + -0.115292676, + 0.047323048, + -0.07599195, + -0.074344486, + -0.029194841, + -0.020079462, + -0.034749795, + -0.05769437, + -0.0301632, + 0.04749987, + 0.012206333, + 0.011497502, + -0.051970575, + 0.05972769, + 0.03281016, + 0.0013676677, + 0.057720944, + -0.041179247, + -0.02150875, + -0.0067487382, + 0.1419711, + 0.05795878, + 0.010094941, + 0.09603845, + 0.014521089, + 0.02133803, + -0.07551916, + 0.07887724, + -0.04273237, + -0.06601746, + -0.038729392, + -0.008161129, + 0.015012324, + -0.049418066, + -0.037083283, + -0.02378242, + 0.03743137, + 0.008194503, + -0.086978436, + -0.05960285, + -0.07732487, + -0.056507926, + 0.029065313, + 0.0073954053, + -0.077878684, + 0.0026059505, + -0.10405392, + -0.04738624, + -0.015872862, + -0.11591199, + 0.09724705, + 0.0049243565, + -0.010273523, + 0.0066429917, + -0.060295314, + 0.02550513, + -0.052950058, + -0.0038489713, + -0.050250847, + 0.07679287, + 0.046089787, + 0.007386997, + 0.0046740095, + 0.07385862, + -0.07792065, + 0.0013675193, + 0.013730894, + 0.05658653, + 0.021934126, + 0.007195913, + 0.0076705213, + 0.10221154, + 0.060060997, + 0.036779005, + -0.037765697, + -1.187368e-08, + -0.00885571, + 0.01760442, + 0.062224448, + 0.032051455, + -0.011581793, + 0.051908698, + -0.011685676, + -0.06391574, + -0.029866237, + 0.03258576, + 0.0055078953, + -0.012040446, + -0.054406017, + -0.056690563, + -0.030638037, + 0.14276367, + 0.028526368, + -0.028743364, + 0.019917691, + 0.025652615, + 0.073813364, + -0.0066998666, + 0.0061508445, + 0.09610696, + -0.08799916, + -0.0089272335, + 0.03823298, + 0.04832936, + 0.018829934, + -0.10534708, + 0.048226915, + -0.02225069, + 0.020491786, + 0.014641141, + 0.030794447, + -0.029119467, + 0.008283775, + -0.04506887, + 0.0025344177, + 0.021756247, + -0.008108281, + 0.00904927, + -0.013340866, + -0.014037631, + 0.06845187, + 0.045173325, + -0.034587316, + -0.07275669, + -0.004159724, + -0.058231864, + -0.033032075, + 0.0040235794, + -0.019985583, + -0.020122562, + 0.055365406, + 0.10250875, + -0.10799118, + -0.013780294, + -0.009652406, + 0.015592658, + -0.031221472, + 0.1329332, + 0.15243866, + -0.022426173 + ] + ] + } + }, + "is_streaming": false + } +} diff --git a/tests/integration/recordings/responses/dbc41d2417e1.json b/tests/integration/recordings/responses/dbc41d2417e1.json new file mode 100644 index 000000000..ce6a7ec62 --- /dev/null +++ b/tests/integration/recordings/responses/dbc41d2417e1.json @@ -0,0 +1,674 @@ +{ + "request": { + "method": "POST", + "url": "http://localhost:11434/v1/chat/completions", + "headers": {}, + "body": { + "model": "llama3.2:3b-instruct-fp16", + "messages": [ + { + "role": "user", + "content": "Hello, world!" + } + ], + "stream": true + }, + "endpoint": "/chat/completions", + "model": "llama3.2:3b-instruct-fp16" + }, + "response": { + "body": [ + { + "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk", + "__data__": { + "id": "chatcmpl-698", + "choices": [ + { + "delta": { + "content": "Hello", + "function_call": null, + "refusal": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1754422171, + "model": "llama3.2:3b-instruct-fp16", + "object": "chat.completion.chunk", + "service_tier": null, + "system_fingerprint": "fp_ollama", + "usage": null + } + }, + { + "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk", + "__data__": { + "id": "chatcmpl-698", + "choices": [ + { + "delta": { + "content": "!", + "function_call": null, + "refusal": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1754422171, + "model": "llama3.2:3b-instruct-fp16", + "object": "chat.completion.chunk", + "service_tier": null, + "system_fingerprint": "fp_ollama", + "usage": null + } + }, + { + "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk", + "__data__": { + "id": "chatcmpl-698", + "choices": [ + { + "delta": { + "content": " It", + "function_call": null, + "refusal": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1754422171, + "model": "llama3.2:3b-instruct-fp16", + "object": "chat.completion.chunk", + "service_tier": null, + "system_fingerprint": "fp_ollama", + "usage": null + } + }, + { + "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk", + "__data__": { + "id": "chatcmpl-698", + "choices": [ + { + "delta": { + "content": "'s", + "function_call": null, + "refusal": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1754422171, + "model": "llama3.2:3b-instruct-fp16", + "object": "chat.completion.chunk", + "service_tier": null, + "system_fingerprint": "fp_ollama", + "usage": null + } + }, + { + "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk", + "__data__": { + "id": "chatcmpl-698", + "choices": [ + { + "delta": { + "content": " nice", + "function_call": null, + "refusal": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1754422171, + "model": "llama3.2:3b-instruct-fp16", + "object": "chat.completion.chunk", + "service_tier": null, + "system_fingerprint": "fp_ollama", + "usage": null + } + }, + { + "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk", + "__data__": { + "id": "chatcmpl-698", + "choices": [ + { + "delta": { + "content": " to", + "function_call": null, + "refusal": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1754422171, + "model": "llama3.2:3b-instruct-fp16", + "object": "chat.completion.chunk", + "service_tier": null, + "system_fingerprint": "fp_ollama", + "usage": null + } + }, + { + "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk", + "__data__": { + "id": "chatcmpl-698", + "choices": [ + { + "delta": { + "content": " meet", + "function_call": null, + "refusal": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1754422171, + "model": "llama3.2:3b-instruct-fp16", + "object": "chat.completion.chunk", + "service_tier": null, + "system_fingerprint": "fp_ollama", + "usage": null + } + }, + { + "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk", + "__data__": { + "id": "chatcmpl-698", + "choices": [ + { + "delta": { + "content": " you", + "function_call": null, + "refusal": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1754422171, + "model": "llama3.2:3b-instruct-fp16", + "object": "chat.completion.chunk", + "service_tier": null, + "system_fingerprint": "fp_ollama", + "usage": null + } + }, + { + "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk", + "__data__": { + "id": "chatcmpl-698", + "choices": [ + { + "delta": { + "content": ".", + "function_call": null, + "refusal": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1754422171, + "model": "llama3.2:3b-instruct-fp16", + "object": "chat.completion.chunk", + "service_tier": null, + "system_fingerprint": "fp_ollama", + "usage": null + } + }, + { + "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk", + "__data__": { + "id": "chatcmpl-698", + "choices": [ + { + "delta": { + "content": " Is", + "function_call": null, + "refusal": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1754422171, + "model": "llama3.2:3b-instruct-fp16", + "object": "chat.completion.chunk", + "service_tier": null, + "system_fingerprint": "fp_ollama", + "usage": null + } + }, + { + "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk", + "__data__": { + "id": "chatcmpl-698", + "choices": [ + { + "delta": { + "content": " there", + "function_call": null, + "refusal": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1754422171, + "model": "llama3.2:3b-instruct-fp16", + "object": "chat.completion.chunk", + "service_tier": null, + "system_fingerprint": "fp_ollama", + "usage": null + } + }, + { + "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk", + "__data__": { + "id": "chatcmpl-698", + "choices": [ + { + "delta": { + "content": " something", + "function_call": null, + "refusal": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1754422171, + "model": "llama3.2:3b-instruct-fp16", + "object": "chat.completion.chunk", + "service_tier": null, + "system_fingerprint": "fp_ollama", + "usage": null + } + }, + { + "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk", + "__data__": { + "id": "chatcmpl-698", + "choices": [ + { + "delta": { + "content": " I", + "function_call": null, + "refusal": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1754422171, + "model": "llama3.2:3b-instruct-fp16", + "object": "chat.completion.chunk", + "service_tier": null, + "system_fingerprint": "fp_ollama", + "usage": null + } + }, + { + "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk", + "__data__": { + "id": "chatcmpl-698", + "choices": [ + { + "delta": { + "content": " can", + "function_call": null, + "refusal": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1754422171, + "model": "llama3.2:3b-instruct-fp16", + "object": "chat.completion.chunk", + "service_tier": null, + "system_fingerprint": "fp_ollama", + "usage": null + } + }, + { + "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk", + "__data__": { + "id": "chatcmpl-698", + "choices": [ + { + "delta": { + "content": " help", + "function_call": null, + "refusal": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1754422172, + "model": "llama3.2:3b-instruct-fp16", + "object": "chat.completion.chunk", + "service_tier": null, + "system_fingerprint": "fp_ollama", + "usage": null + } + }, + { + "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk", + "__data__": { + "id": "chatcmpl-698", + "choices": [ + { + "delta": { + "content": " you", + "function_call": null, + "refusal": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1754422172, + "model": "llama3.2:3b-instruct-fp16", + "object": "chat.completion.chunk", + "service_tier": null, + "system_fingerprint": "fp_ollama", + "usage": null + } + }, + { + "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk", + "__data__": { + "id": "chatcmpl-698", + "choices": [ + { + "delta": { + "content": " with", + "function_call": null, + "refusal": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1754422172, + "model": "llama3.2:3b-instruct-fp16", + "object": "chat.completion.chunk", + "service_tier": null, + "system_fingerprint": "fp_ollama", + "usage": null + } + }, + { + "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk", + "__data__": { + "id": "chatcmpl-698", + "choices": [ + { + "delta": { + "content": " or", + "function_call": null, + "refusal": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1754422172, + "model": "llama3.2:3b-instruct-fp16", + "object": "chat.completion.chunk", + "service_tier": null, + "system_fingerprint": "fp_ollama", + "usage": null + } + }, + { + "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk", + "__data__": { + "id": "chatcmpl-698", + "choices": [ + { + "delta": { + "content": " would", + "function_call": null, + "refusal": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1754422172, + "model": "llama3.2:3b-instruct-fp16", + "object": "chat.completion.chunk", + "service_tier": null, + "system_fingerprint": "fp_ollama", + "usage": null + } + }, + { + "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk", + "__data__": { + "id": "chatcmpl-698", + "choices": [ + { + "delta": { + "content": " you", + "function_call": null, + "refusal": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1754422172, + "model": "llama3.2:3b-instruct-fp16", + "object": "chat.completion.chunk", + "service_tier": null, + "system_fingerprint": "fp_ollama", + "usage": null + } + }, + { + "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk", + "__data__": { + "id": "chatcmpl-698", + "choices": [ + { + "delta": { + "content": " like", + "function_call": null, + "refusal": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1754422172, + "model": "llama3.2:3b-instruct-fp16", + "object": "chat.completion.chunk", + "service_tier": null, + "system_fingerprint": "fp_ollama", + "usage": null + } + }, + { + "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk", + "__data__": { + "id": "chatcmpl-698", + "choices": [ + { + "delta": { + "content": " to", + "function_call": null, + "refusal": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1754422172, + "model": "llama3.2:3b-instruct-fp16", + "object": "chat.completion.chunk", + "service_tier": null, + "system_fingerprint": "fp_ollama", + "usage": null + } + }, + { + "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk", + "__data__": { + "id": "chatcmpl-698", + "choices": [ + { + "delta": { + "content": " chat", + "function_call": null, + "refusal": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1754422172, + "model": "llama3.2:3b-instruct-fp16", + "object": "chat.completion.chunk", + "service_tier": null, + "system_fingerprint": "fp_ollama", + "usage": null + } + }, + { + "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk", + "__data__": { + "id": "chatcmpl-698", + "choices": [ + { + "delta": { + "content": "?", + "function_call": null, + "refusal": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": null, + "index": 0, + "logprobs": null + } + ], + "created": 1754422172, + "model": "llama3.2:3b-instruct-fp16", + "object": "chat.completion.chunk", + "service_tier": null, + "system_fingerprint": "fp_ollama", + "usage": null + } + }, + { + "__type__": "openai.types.chat.chat_completion_chunk.ChatCompletionChunk", + "__data__": { + "id": "chatcmpl-698", + "choices": [ + { + "delta": { + "content": "", + "function_call": null, + "refusal": null, + "role": "assistant", + "tool_calls": null + }, + "finish_reason": "stop", + "index": 0, + "logprobs": null + } + ], + "created": 1754422172, + "model": "llama3.2:3b-instruct-fp16", + "object": "chat.completion.chunk", + "service_tier": null, + "system_fingerprint": "fp_ollama", + "usage": null + } + } + ], + "is_streaming": true + } +} diff --git a/tests/integration/recordings/responses/f1ea938b0b0d.json b/tests/integration/recordings/responses/f1ea938b0b0d.json new file mode 100644 index 000000000..da846a30b --- /dev/null +++ b/tests/integration/recordings/responses/f1ea938b0b0d.json @@ -0,0 +1,56 @@ +{ + "request": { + "method": "POST", + "url": "http://localhost:11434/v1/chat/completions", + "headers": {}, + "body": { + "model": "llama3.2:3b-instruct-fp16", + "messages": [ + { + "role": "user", + "content": "Hello, world!" + } + ], + "stream": false + }, + "endpoint": "/chat/completions", + "model": "llama3.2:3b-instruct-fp16" + }, + "response": { + "body": { + "__type__": "openai.types.chat.chat_completion.ChatCompletion", + "__data__": { + "id": "chatcmpl-796", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "message": { + "content": "Hello! It's nice to meet you. Is there something I can help you with, or would you like to chat?", + "refusal": null, + "role": "assistant", + "annotations": null, + "audio": null, + "function_call": null, + "tool_calls": null + } + } + ], + "created": 1754422173, + "model": "llama3.2:3b-instruct-fp16", + "object": "chat.completion", + "service_tier": null, + "system_fingerprint": "fp_ollama", + "usage": { + "completion_tokens": 26, + "prompt_tokens": 29, + "total_tokens": 55, + "completion_tokens_details": null, + "prompt_tokens_details": null + } + } + }, + "is_streaming": false + } +} diff --git a/tests/integration/vector_io/test_openai_vector_stores.py b/tests/integration/vector_io/test_openai_vector_stores.py index a83b30728..1c9ef92b6 100644 --- a/tests/integration/vector_io/test_openai_vector_stores.py +++ b/tests/integration/vector_io/test_openai_vector_stores.py @@ -11,10 +11,8 @@ from io import BytesIO import pytest from llama_stack_client import BadRequestError, LlamaStackClient from openai import BadRequestError as OpenAIBadRequestError -from openai import OpenAI from llama_stack.apis.vector_io import Chunk -from llama_stack.core.library_client import LlamaStackAsLibraryClient logger = logging.getLogger(__name__) @@ -69,19 +67,6 @@ def skip_if_provider_doesnt_support_openai_vector_stores_search(client_with_mode ) -@pytest.fixture -def openai_client(client_with_models): - base_url = f"{client_with_models.base_url}/v1/openai/v1" - return OpenAI(base_url=base_url, api_key="fake") - - -@pytest.fixture(params=["openai_client", "llama_stack_client"]) -def compat_client(request, client_with_models): - if request.param == "openai_client" and isinstance(client_with_models, LlamaStackAsLibraryClient): - pytest.skip("OpenAI client tests not supported with library client") - return request.getfixturevalue(request.param) - - @pytest.fixture(scope="session") def sample_chunks(): return [