phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-06-28 19:04:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

migrate dataset to resource (#420)

Browse source 
* migrate dataset to resource

* remove auto discovery

* remove listing of providers's datasets

* fix after rebase

---------

Co-authored-by: Dinesh Yeduguru <dineshyv@fb.com>
This commit is contained in:
Dinesh Yeduguru 2024-11-11 17:14:41 -08:00 • committed by GitHub
parent 38cce97597
commit b95cb5308f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 71 additions and 62 deletions
Download patch file Download diff file Expand all files Collapse all files

9
llama_stack/providers/adapters/datasetio/huggingface/huggingface.py
View file

@ -3,7 +3,7 @@
#
# This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree.
from typing import List, Optional
from typing import Optional
from llama_stack.apis.datasetio import * # noqa: F403
@ -15,7 +15,7 @@ from llama_stack.providers.utils.datasetio.url_utils import get_dataframe_from_u
from .config import HuggingfaceDatasetIOConfig
def load_hf_dataset(dataset_def: DatasetDef):
def load_hf_dataset(dataset_def: Dataset):
if dataset_def.metadata.get("path", None):
return hf_datasets.load_dataset(**dataset_def.metadata)
@ -41,13 +41,10 @@ class HuggingfaceDatasetIOImpl(DatasetIO, DatasetsProtocolPrivate):
async def register_dataset(
self,
dataset_def: DatasetDef,
dataset_def: Dataset,
) -> None:
self.dataset_infos[dataset_def.identifier] = dataset_def
async def list_datasets(self) -> List[DatasetDef]:
return list(self.dataset_infos.values())
async def get_rows_paginated(
self,
dataset_id: str,