From 871802f489301621996c8a30abd56fc9674923e2 Mon Sep 17 00:00:00 2001 From: Charlie Doern Date: Wed, 17 Sep 2025 12:43:13 -0400 Subject: [PATCH] feat(api): level v1beta APIs level the following APIs as v1beta: 1. eval: job scheduling is not implemented. Relies heavily on the datasetio API which is under development/missing routes. 2. datasetio: used primarily by eval and training. Given that training is v1alpha, and eval is v1beta, datasetio is likely to change in structure as real usages of the API spin up. Register,unregister, and iter dataset is sparsely implemented meaning the shape of that route is likely to change. 3. telemetry: telemetry has been going through many changes. for example query_metrics was not even implemented until recently and had to change its shape to work. putting this in v1beta will allow us to fix functionality like OTEL, sqlite, etc. The routes themselves are set, but the structure might change a bit Signed-off-by: Charlie Doern --- docs/_static/llama-stack-spec.html | 32 +++++++++---------- docs/_static/llama-stack-spec.yaml | 32 +++++++++---------- llama_stack/apis/datasetio/datasetio.py | 4 +-- llama_stack/apis/datasets/datasets.py | 8 ++--- llama_stack/apis/eval/eval.py | 10 +++--- llama_stack/apis/telemetry/telemetry.py | 21 +++++++----- .../remote/datasetio/nvidia/datasetio.py | 4 +-- tests/unit/providers/nvidia/test_datastore.py | 6 ++-- 8 files changed, 61 insertions(+), 56 deletions(-) diff --git a/docs/_static/llama-stack-spec.html b/docs/_static/llama-stack-spec.html index 63c68d835..a37da024b 100644 --- a/docs/_static/llama-stack-spec.html +++ b/docs/_static/llama-stack-spec.html @@ -40,7 +40,7 @@ } ], "paths": { - "/v1/datasetio/append-rows/{dataset_id}": { + "/v1beta/datasetio/append-rows/{dataset_id}": { "post": { "responses": { "200": { @@ -1155,7 +1155,7 @@ } } }, - "/v1/eval/benchmarks/{benchmark_id}/evaluations": { + "/v1beta/eval/benchmarks/{benchmark_id}/evaluations": { "post": { "responses": { "200": { @@ -1459,7 +1459,7 @@ ] } }, - "/v1/datasets/{dataset_id}": { + "/v1beta/datasets/{dataset_id}": { "get": { "responses": { "200": { @@ -1767,7 +1767,7 @@ ] } }, - "/v1/telemetry/traces/{trace_id}/spans/{span_id}": { + "/v1beta/telemetry/traces/{trace_id}/spans/{span_id}": { "get": { "responses": { "200": { @@ -1819,7 +1819,7 @@ ] } }, - "/v1/telemetry/spans/{span_id}/tree": { + "/v1beta/telemetry/spans/{span_id}/tree": { "post": { "responses": { "200": { @@ -1992,7 +1992,7 @@ ] } }, - "/v1/telemetry/traces/{trace_id}": { + "/v1beta/telemetry/traces/{trace_id}": { "get": { "responses": { "200": { @@ -2422,7 +2422,7 @@ } } }, - "/v1/datasetio/iterrows/{dataset_id}": { + "/v1beta/datasetio/iterrows/{dataset_id}": { "get": { "responses": { "200": { @@ -2483,7 +2483,7 @@ ] } }, - "/v1/eval/benchmarks/{benchmark_id}/jobs/{job_id}": { + "/v1beta/eval/benchmarks/{benchmark_id}/jobs/{job_id}": { "get": { "responses": { "200": { @@ -2578,7 +2578,7 @@ ] } }, - "/v1/eval/benchmarks/{benchmark_id}/jobs/{job_id}/result": { + "/v1beta/eval/benchmarks/{benchmark_id}/jobs/{job_id}/result": { "get": { "responses": { "200": { @@ -2876,7 +2876,7 @@ } } }, - "/v1/datasets": { + "/v1beta/datasets": { "get": { "responses": { "200": { @@ -3601,7 +3601,7 @@ } } }, - "/v1/telemetry/events": { + "/v1beta/telemetry/events": { "post": { "responses": { "200": { @@ -4810,7 +4810,7 @@ } } }, - "/v1/telemetry/metrics/{metric_name}": { + "/v1beta/telemetry/metrics/{metric_name}": { "post": { "responses": { "200": { @@ -4863,7 +4863,7 @@ } } }, - "/v1/telemetry/spans": { + "/v1beta/telemetry/spans": { "post": { "responses": { "200": { @@ -4906,7 +4906,7 @@ } } }, - "/v1/telemetry/traces": { + "/v1beta/telemetry/traces": { "post": { "responses": { "200": { @@ -5068,7 +5068,7 @@ } } }, - "/v1/eval/benchmarks/{benchmark_id}/jobs": { + "/v1beta/eval/benchmarks/{benchmark_id}/jobs": { "post": { "responses": { "200": { @@ -5207,7 +5207,7 @@ } } }, - "/v1/telemetry/spans/export": { + "/v1beta/telemetry/spans/export": { "post": { "responses": { "200": { diff --git a/docs/_static/llama-stack-spec.yaml b/docs/_static/llama-stack-spec.yaml index ea7d3248d..0ea382bc6 100644 --- a/docs/_static/llama-stack-spec.yaml +++ b/docs/_static/llama-stack-spec.yaml @@ -10,7 +10,7 @@ info: servers: - url: http://any-hosted-llama-stack.com paths: - /v1/datasetio/append-rows/{dataset_id}: + /v1beta/datasetio/append-rows/{dataset_id}: post: responses: '200': @@ -798,7 +798,7 @@ paths: schema: $ref: '#/components/schemas/EmbeddingsRequest' required: true - /v1/eval/benchmarks/{benchmark_id}/evaluations: + /v1beta/eval/benchmarks/{benchmark_id}/evaluations: post: responses: '200': @@ -1007,7 +1007,7 @@ paths: required: true schema: type: string - /v1/datasets/{dataset_id}: + /v1beta/datasets/{dataset_id}: get: responses: '200': @@ -1222,7 +1222,7 @@ paths: required: true schema: type: string - /v1/telemetry/traces/{trace_id}/spans/{span_id}: + /v1beta/telemetry/traces/{trace_id}/spans/{span_id}: get: responses: '200': @@ -1258,7 +1258,7 @@ paths: required: true schema: type: string - /v1/telemetry/spans/{span_id}/tree: + /v1beta/telemetry/spans/{span_id}/tree: post: responses: '200': @@ -1375,7 +1375,7 @@ paths: required: true schema: type: string - /v1/telemetry/traces/{trace_id}: + /v1beta/telemetry/traces/{trace_id}: get: responses: '200': @@ -1678,7 +1678,7 @@ paths: schema: $ref: '#/components/schemas/InvokeToolRequest' required: true - /v1/datasetio/iterrows/{dataset_id}: + /v1beta/datasetio/iterrows/{dataset_id}: get: responses: '200': @@ -1735,7 +1735,7 @@ paths: required: false schema: type: integer - /v1/eval/benchmarks/{benchmark_id}/jobs/{job_id}: + /v1beta/eval/benchmarks/{benchmark_id}/jobs/{job_id}: get: responses: '200': @@ -1802,7 +1802,7 @@ paths: required: true schema: type: string - /v1/eval/benchmarks/{benchmark_id}/jobs/{job_id}/result: + /v1beta/eval/benchmarks/{benchmark_id}/jobs/{job_id}/result: get: responses: '200': @@ -2010,7 +2010,7 @@ paths: schema: $ref: '#/components/schemas/OpenaiChatCompletionRequest' required: true - /v1/datasets: + /v1beta/datasets: get: responses: '200': @@ -2524,7 +2524,7 @@ paths: schema: $ref: '#/components/schemas/RegisterVectorDbRequest' required: true - /v1/telemetry/events: + /v1beta/telemetry/events: post: responses: '200': @@ -3414,7 +3414,7 @@ paths: schema: $ref: '#/components/schemas/QueryChunksRequest' required: true - /v1/telemetry/metrics/{metric_name}: + /v1beta/telemetry/metrics/{metric_name}: post: responses: '200': @@ -3449,7 +3449,7 @@ paths: schema: $ref: '#/components/schemas/QueryMetricsRequest' required: true - /v1/telemetry/spans: + /v1beta/telemetry/spans: post: responses: '200': @@ -3478,7 +3478,7 @@ paths: schema: $ref: '#/components/schemas/QuerySpansRequest' required: true - /v1/telemetry/traces: + /v1beta/telemetry/traces: post: responses: '200': @@ -3595,7 +3595,7 @@ paths: schema: $ref: '#/components/schemas/ResumeAgentTurnRequest' required: true - /v1/eval/benchmarks/{benchmark_id}/jobs: + /v1beta/eval/benchmarks/{benchmark_id}/jobs: post: responses: '200': @@ -3691,7 +3691,7 @@ paths: schema: $ref: '#/components/schemas/RunShieldRequest' required: true - /v1/telemetry/spans/export: + /v1beta/telemetry/spans/export: post: responses: '200': diff --git a/llama_stack/apis/datasetio/datasetio.py b/llama_stack/apis/datasetio/datasetio.py index 1183983cc..449bd22d7 100644 --- a/llama_stack/apis/datasetio/datasetio.py +++ b/llama_stack/apis/datasetio/datasetio.py @@ -20,7 +20,7 @@ class DatasetIO(Protocol): # keeping for aligning with inference/safety, but this is not used dataset_store: DatasetStore - @webmethod(route="/datasetio/iterrows/{dataset_id:path}", method="GET") + @webmethod(route="/datasetio/iterrows/{dataset_id:path}", method="GET", level="v1beta") async def iterrows( self, dataset_id: str, @@ -44,7 +44,7 @@ class DatasetIO(Protocol): """ ... - @webmethod(route="/datasetio/append-rows/{dataset_id:path}", method="POST") + @webmethod(route="/datasetio/append-rows/{dataset_id:path}", method="POST", level="v1beta") async def append_rows(self, dataset_id: str, rows: list[dict[str, Any]]) -> None: """Append rows to a dataset. diff --git a/llama_stack/apis/datasets/datasets.py b/llama_stack/apis/datasets/datasets.py index f347e0e29..1d5869455 100644 --- a/llama_stack/apis/datasets/datasets.py +++ b/llama_stack/apis/datasets/datasets.py @@ -145,7 +145,7 @@ class ListDatasetsResponse(BaseModel): class Datasets(Protocol): - @webmethod(route="/datasets", method="POST") + @webmethod(route="/datasets", method="POST", level="v1beta") async def register_dataset( self, purpose: DatasetPurpose, @@ -214,7 +214,7 @@ class Datasets(Protocol): """ ... - @webmethod(route="/datasets/{dataset_id:path}", method="GET") + @webmethod(route="/datasets/{dataset_id:path}", method="GET", level="v1beta") async def get_dataset( self, dataset_id: str, @@ -226,7 +226,7 @@ class Datasets(Protocol): """ ... - @webmethod(route="/datasets", method="GET") + @webmethod(route="/datasets", method="GET", level="v1beta") async def list_datasets(self) -> ListDatasetsResponse: """List all datasets. @@ -234,7 +234,7 @@ class Datasets(Protocol): """ ... - @webmethod(route="/datasets/{dataset_id:path}", method="DELETE") + @webmethod(route="/datasets/{dataset_id:path}", method="DELETE", level="v1beta") async def unregister_dataset( self, dataset_id: str, diff --git a/llama_stack/apis/eval/eval.py b/llama_stack/apis/eval/eval.py index 83a0a8e56..ea3a72713 100644 --- a/llama_stack/apis/eval/eval.py +++ b/llama_stack/apis/eval/eval.py @@ -83,7 +83,7 @@ class EvaluateResponse(BaseModel): class Eval(Protocol): """Llama Stack Evaluation API for running evaluations on model and agent candidates.""" - @webmethod(route="/eval/benchmarks/{benchmark_id}/jobs", method="POST") + @webmethod(route="/eval/benchmarks/{benchmark_id}/jobs", method="POST", level="v1beta") async def run_eval( self, benchmark_id: str, @@ -97,7 +97,7 @@ class Eval(Protocol): """ ... - @webmethod(route="/eval/benchmarks/{benchmark_id}/evaluations", method="POST") + @webmethod(route="/eval/benchmarks/{benchmark_id}/evaluations", method="POST", level="v1beta") async def evaluate_rows( self, benchmark_id: str, @@ -115,7 +115,7 @@ class Eval(Protocol): """ ... - @webmethod(route="/eval/benchmarks/{benchmark_id}/jobs/{job_id}", method="GET") + @webmethod(route="/eval/benchmarks/{benchmark_id}/jobs/{job_id}", method="GET", level="v1beta") async def job_status(self, benchmark_id: str, job_id: str) -> Job: """Get the status of a job. @@ -125,7 +125,7 @@ class Eval(Protocol): """ ... - @webmethod(route="/eval/benchmarks/{benchmark_id}/jobs/{job_id}", method="DELETE") + @webmethod(route="/eval/benchmarks/{benchmark_id}/jobs/{job_id}", method="DELETE", level="v1beta") async def job_cancel(self, benchmark_id: str, job_id: str) -> None: """Cancel a job. @@ -134,7 +134,7 @@ class Eval(Protocol): """ ... - @webmethod(route="/eval/benchmarks/{benchmark_id}/jobs/{job_id}/result", method="GET") + @webmethod(route="/eval/benchmarks/{benchmark_id}/jobs/{job_id}/result", method="GET", level="v1beta") async def job_result(self, benchmark_id: str, job_id: str) -> EvaluateResponse: """Get the result of a job. diff --git a/llama_stack/apis/telemetry/telemetry.py b/llama_stack/apis/telemetry/telemetry.py index 8d1b5d697..b309ebb0a 100644 --- a/llama_stack/apis/telemetry/telemetry.py +++ b/llama_stack/apis/telemetry/telemetry.py @@ -412,7 +412,7 @@ class QueryMetricsResponse(BaseModel): @runtime_checkable class Telemetry(Protocol): - @webmethod(route="/telemetry/events", method="POST") + @webmethod(route="/telemetry/events", method="POST", level="v1beta") async def log_event( self, event: Event, @@ -425,7 +425,7 @@ class Telemetry(Protocol): """ ... - @webmethod(route="/telemetry/traces", method="POST", required_scope=REQUIRED_SCOPE) + @webmethod(route="/telemetry/traces", method="POST", required_scope=REQUIRED_SCOPE, level="v1beta") async def query_traces( self, attribute_filters: list[QueryCondition] | None = None, @@ -443,7 +443,7 @@ class Telemetry(Protocol): """ ... - @webmethod(route="/telemetry/traces/{trace_id:path}", method="GET", required_scope=REQUIRED_SCOPE) + @webmethod(route="/telemetry/traces/{trace_id:path}", method="GET", required_scope=REQUIRED_SCOPE, level="v1beta") async def get_trace(self, trace_id: str) -> Trace: """Get a trace by its ID. @@ -453,7 +453,10 @@ class Telemetry(Protocol): ... @webmethod( - route="/telemetry/traces/{trace_id:path}/spans/{span_id:path}", method="GET", required_scope=REQUIRED_SCOPE + route="/telemetry/traces/{trace_id:path}/spans/{span_id:path}", + method="GET", + required_scope=REQUIRED_SCOPE, + level="v1beta", ) async def get_span(self, trace_id: str, span_id: str) -> Span: """Get a span by its ID. @@ -464,7 +467,9 @@ class Telemetry(Protocol): """ ... - @webmethod(route="/telemetry/spans/{span_id:path}/tree", method="POST", required_scope=REQUIRED_SCOPE) + @webmethod( + route="/telemetry/spans/{span_id:path}/tree", method="POST", required_scope=REQUIRED_SCOPE, level="v1beta" + ) async def get_span_tree( self, span_id: str, @@ -480,7 +485,7 @@ class Telemetry(Protocol): """ ... - @webmethod(route="/telemetry/spans", method="POST", required_scope=REQUIRED_SCOPE) + @webmethod(route="/telemetry/spans", method="POST", required_scope=REQUIRED_SCOPE, level="v1beta") async def query_spans( self, attribute_filters: list[QueryCondition], @@ -496,7 +501,7 @@ class Telemetry(Protocol): """ ... - @webmethod(route="/telemetry/spans/export", method="POST") + @webmethod(route="/telemetry/spans/export", method="POST", level="v1beta") async def save_spans_to_dataset( self, attribute_filters: list[QueryCondition], @@ -513,7 +518,7 @@ class Telemetry(Protocol): """ ... - @webmethod(route="/telemetry/metrics/{metric_name}", method="POST", required_scope=REQUIRED_SCOPE) + @webmethod(route="/telemetry/metrics/{metric_name}", method="POST", required_scope=REQUIRED_SCOPE, level="v1beta") async def query_metrics( self, metric_name: str, diff --git a/llama_stack/providers/remote/datasetio/nvidia/datasetio.py b/llama_stack/providers/remote/datasetio/nvidia/datasetio.py index f723c92cc..948d11055 100644 --- a/llama_stack/providers/remote/datasetio/nvidia/datasetio.py +++ b/llama_stack/providers/remote/datasetio/nvidia/datasetio.py @@ -78,7 +78,7 @@ class NvidiaDatasetIOAdapter: request_body["description"] = dataset_def.metadata.get("description") await self._make_request( "POST", - "/v1/datasets", + "/v1beta/datasets", json=request_body, ) return dataset_def @@ -100,7 +100,7 @@ class NvidiaDatasetIOAdapter: ) -> None: await self._make_request( "DELETE", - f"/v1/datasets/{self.config.dataset_namespace}/{dataset_id}", + f"/v1beta/datasets/{self.config.dataset_namespace}/{dataset_id}", headers={"Accept": "application/json", "Content-Type": "application/json"}, ) diff --git a/tests/unit/providers/nvidia/test_datastore.py b/tests/unit/providers/nvidia/test_datastore.py index b59636f7b..bf146adb2 100644 --- a/tests/unit/providers/nvidia/test_datastore.py +++ b/tests/unit/providers/nvidia/test_datastore.py @@ -67,7 +67,7 @@ def test_register_dataset(nvidia_adapter, run_async): _assert_request( mock_make_request, "POST", - "/v1/datasets", + "/v1beta/datasets", expected_json={ "name": "test-dataset", "namespace": "default", @@ -91,7 +91,7 @@ def test_unregister_dataset(nvidia_adapter, run_async): run_async(adapter.unregister_dataset(dataset_id)) mock_make_request.assert_called_once() - _assert_request(mock_make_request, "DELETE", "/v1/datasets/default/test-dataset") + _assert_request(mock_make_request, "DELETE", "/v1beta/datasets/default/test-dataset") def test_register_dataset_with_custom_namespace_project(run_async): @@ -130,7 +130,7 @@ def test_register_dataset_with_custom_namespace_project(run_async): _assert_request( mock_make_request, "POST", - "/v1/datasets", + "/v1beta/datasets", expected_json={ "name": "test-dataset", "namespace": "custom-namespace",