mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-06-27 18:50:41 +00:00
5287b437ae
## PR Stack - https://github.com/meta-llama/llama-stack/pull/1573 - https://github.com/meta-llama/llama-stack/pull/1625 - https://github.com/meta-llama/llama-stack/pull/1656 - https://github.com/meta-llama/llama-stack/pull/1657 - https://github.com/meta-llama/llama-stack/pull/1658 - https://github.com/meta-llama/llama-stack/pull/1659 - https://github.com/meta-llama/llama-stack/pull/1660 **Client SDK** - https://github.com/meta-llama/llama-stack-client-python/pull/203 **CI** -1391130488<img width="1042" alt="image" src="https://github.com/user-attachments/assets/69636067-376d-436b-9204-896e2dd490ca" /> -- the test_rag_agent_with_attachments is flaky and not related to this PR ## Doc <img width="789" alt="image" src="https://github.com/user-attachments/assets/b88390f3-73d6-4483-b09a-a192064e32d9" /> ## Client Usage ```python client.datasets.register( source={ "type": "uri", "uri": "lsfs://mydata.jsonl", }, schema="jsonl_messages", # optional dataset_id="my_first_train_data" ) # quick prototype debugging client.datasets.register( data_reference={ "type": "rows", "rows": [ "messages": [...], ], }, schema="jsonl_messages", ) ``` ## Test Plan - CI:1387805545``` LLAMA_STACK_CONFIG=fireworks pytest -v tests/integration/datasets/test_datasets.py ``` ``` LLAMA_STACK_CONFIG=fireworks pytest -v tests/integration/scoring/test_scoring.py ``` ``` pytest -v -s --nbval-lax ./docs/notebooks/Llama_Stack_Benchmark_Evals.ipynb ```
54 lines
1.8 KiB
Python
54 lines
1.8 KiB
Python
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
# All rights reserved.
|
|
#
|
|
# This source code is licensed under the terms described in the LICENSE file in
|
|
# the root directory of this source tree.
|
|
|
|
from typing import Any, Dict, List, Optional, Protocol, runtime_checkable
|
|
|
|
from pydantic import BaseModel
|
|
|
|
from llama_stack.apis.datasets import Dataset
|
|
from llama_stack.schema_utils import json_schema_type, webmethod
|
|
|
|
|
|
@json_schema_type
|
|
class IterrowsResponse(BaseModel):
|
|
"""
|
|
A paginated list of rows from a dataset.
|
|
|
|
:param data: The rows in the current page.
|
|
:param next_start_index: Index into dataset for the first row in the next page. None if there are no more rows.
|
|
"""
|
|
|
|
data: List[Dict[str, Any]]
|
|
next_start_index: Optional[int] = None
|
|
|
|
|
|
class DatasetStore(Protocol):
|
|
def get_dataset(self, dataset_id: str) -> Dataset: ...
|
|
|
|
|
|
@runtime_checkable
|
|
class DatasetIO(Protocol):
|
|
# keeping for aligning with inference/safety, but this is not used
|
|
dataset_store: DatasetStore
|
|
|
|
# TODO(xiyan): there's a flakiness here where setting route to "/datasets/" here will not result in proper routing
|
|
@webmethod(route="/datasetio/iterrows/{dataset_id:path}", method="GET")
|
|
async def iterrows(
|
|
self,
|
|
dataset_id: str,
|
|
start_index: Optional[int] = None,
|
|
limit: Optional[int] = None,
|
|
) -> IterrowsResponse:
|
|
"""Get a paginated list of rows from a dataset. Uses cursor-based pagination.
|
|
|
|
:param dataset_id: The ID of the dataset to get the rows from.
|
|
:param start_index: Index into dataset for the first row to get. Get all rows if None.
|
|
:param limit: The number of rows to get.
|
|
"""
|
|
...
|
|
|
|
@webmethod(route="/datasetio/append-rows/{dataset_id:path}", method="POST")
|
|
async def append_rows(self, dataset_id: str, rows: List[Dict[str, Any]]) -> None: ...
|