mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-10-04 04:04:14 +00:00
cffc4edf47
Some checks failed
Integration Auth Tests / test-matrix (oauth2_token) (push) Failing after 4s
Test External Providers Installed via Module / test-external-providers-from-module (venv) (push) Has been skipped
Integration Tests (Replay) / Integration Tests (, , , client=, vision=) (push) Failing after 0s
Test Llama Stack Build / build-single-provider (push) Failing after 2s
Pre-commit / pre-commit (push) Failing after 4s
SqlStore Integration Tests / test-postgres (3.13) (push) Failing after 5s
Test Llama Stack Build / build-ubi9-container-distribution (push) Failing after 3s
Test Llama Stack Build / generate-matrix (push) Failing after 5s
Test Llama Stack Build / build (push) Has been skipped
Vector IO Integration Tests / test-matrix (push) Failing after 6s
Test Llama Stack Build / build-custom-container-distribution (push) Failing after 5s
Python Package Build Test / build (3.13) (push) Failing after 4s
Test External API and Providers / test-external (venv) (push) Failing after 4s
Unit Tests / unit-tests (3.12) (push) Failing after 4s
Update ReadTheDocs / update-readthedocs (push) Failing after 4s
Python Package Build Test / build (3.12) (push) Failing after 7s
Unit Tests / unit-tests (3.13) (push) Failing after 5s
UI Tests / ui-tests (22) (push) Failing after 6s
SqlStore Integration Tests / test-postgres (3.12) (push) Failing after 14s
Implements optional idempotency for batch creation using `idem_tok` parameter: * **Core idempotency**: Same token + parameters returns existing batch * **Conflict detection**: Same token + different parameters raises HTTP 409 ConflictError * **Metadata order independence**: Different key ordering doesn't affect idempotency **API changes:** - Add optional `idem_tok` parameter to `create_batch()` method - Enhanced API documentation with idempotency extensions **Implementation:** - Reference provider supports idempotent batch creation - ConflictError for proper HTTP 409 status code mapping - Comprehensive parameter validation **Testing:** - Unit tests: focused tests covering core scenarios with parametrized conflict detection - Integration tests: tests validating real OpenAI client behavior This enables client-side retry safety and prevents duplicate batch creation when using the same idempotency token, following REST API closes #3144
91 lines
3.7 KiB
Python
91 lines
3.7 KiB
Python
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
# All rights reserved.
|
|
#
|
|
# This source code is licensed under the terms described in the LICENSE file in
|
|
# the root directory of this source tree.
|
|
|
|
"""
|
|
Integration tests for batch idempotency functionality using the OpenAI client library.
|
|
|
|
This module tests the idempotency feature in the batches API using the OpenAI-compatible
|
|
client interface. These tests verify that the idempotency key (idempotency_key) works correctly
|
|
in a real client-server environment.
|
|
|
|
Test Categories:
|
|
1. Successful Idempotency: Same key returns same batch with identical parameters
|
|
- test_idempotent_batch_creation_successful: Verifies that requests with the same
|
|
idempotency key return identical batches, even with different metadata order
|
|
|
|
2. Conflict Detection: Same key with conflicting parameters raises HTTP 409 errors
|
|
- test_idempotency_conflict_with_different_params: Verifies that reusing an idempotency key
|
|
with truly conflicting parameters (both file ID and metadata values) raises ConflictError
|
|
"""
|
|
|
|
import time
|
|
|
|
import pytest
|
|
from openai import ConflictError
|
|
|
|
|
|
class TestBatchesIdempotencyIntegration:
|
|
"""Integration tests for batch idempotency using OpenAI client."""
|
|
|
|
def test_idempotent_batch_creation_successful(self, openai_client):
|
|
"""Test that identical requests with same idempotency key return the same batch."""
|
|
batch1 = openai_client.batches.create(
|
|
input_file_id="bogus-id",
|
|
endpoint="/v1/chat/completions",
|
|
completion_window="24h",
|
|
metadata={
|
|
"test_type": "idempotency_success",
|
|
"purpose": "integration_test",
|
|
},
|
|
extra_body={"idempotency_key": "test-idempotency-token-1"},
|
|
)
|
|
|
|
# sleep to ensure different timestamps
|
|
time.sleep(1)
|
|
|
|
batch2 = openai_client.batches.create(
|
|
input_file_id="bogus-id",
|
|
endpoint="/v1/chat/completions",
|
|
completion_window="24h",
|
|
metadata={
|
|
"purpose": "integration_test",
|
|
"test_type": "idempotency_success",
|
|
}, # Different order
|
|
extra_body={"idempotency_key": "test-idempotency-token-1"},
|
|
)
|
|
|
|
assert batch1.id == batch2.id
|
|
assert batch1.input_file_id == batch2.input_file_id
|
|
assert batch1.endpoint == batch2.endpoint
|
|
assert batch1.completion_window == batch2.completion_window
|
|
assert batch1.metadata == batch2.metadata
|
|
assert batch1.created_at == batch2.created_at
|
|
|
|
def test_idempotency_conflict_with_different_params(self, openai_client):
|
|
"""Test that using same idempotency key with different params raises conflict error."""
|
|
batch1 = openai_client.batches.create(
|
|
input_file_id="bogus-id-1",
|
|
endpoint="/v1/chat/completions",
|
|
completion_window="24h",
|
|
metadata={"test_type": "conflict_test_1"},
|
|
extra_body={"idempotency_key": "conflict-token"},
|
|
)
|
|
|
|
with pytest.raises(ConflictError) as exc_info:
|
|
openai_client.batches.create(
|
|
input_file_id="bogus-id-2", # Different file ID
|
|
endpoint="/v1/chat/completions",
|
|
completion_window="24h",
|
|
metadata={"test_type": "conflict_test_2"}, # Different metadata
|
|
extra_body={"idempotency_key": "conflict-token"}, # Same token
|
|
)
|
|
|
|
assert exc_info.value.status_code == 409
|
|
assert "conflict" in str(exc_info.value).lower()
|
|
|
|
retrieved_batch = openai_client.batches.retrieve(batch1.id)
|
|
assert retrieved_batch.id == batch1.id
|
|
assert retrieved_batch.input_file_id == "bogus-id-1"
|