phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-10-10 13:28:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix(testing): improve api_recorder error messages for missing recordings (#3760)

Browse source 
Replaces opaque error messages when recordings are not found with
somewhat better guidance

Before:
```
No recorded response found for request hash: abc123...
To record this response, run with LLAMA_STACK_TEST_INFERENCE_MODE=record
```

After:
```
Recording not found for request hash: abc123
Model: gpt-4 | Request: POST https://api.openai.com/v1/chat/completions

Run './scripts/integration-tests.sh --inference-mode record-if-missing' with required API keys to generate.
```
This commit is contained in:
Ashwin Bharambe 2025-10-09 15:04:16 -07:00 • committed by GitHub
parent a055a32ee4
commit 841d0c3583
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 8 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

14
llama_stack/testing/api_recorder.py
View file

@ -556,9 +556,9 @@ async def _patched_tool_invoke_method(
return recording["response"]["body"]
elif _current_mode == APIRecordingMode.REPLAY:
raise RuntimeError(
f"No recorded tool result found for {provider_name}.{tool_name}\n"
f"Request: {kwargs}\n"
f"To record this response, run with LLAMA_STACK_TEST_INFERENCE_MODE=record"
f"Recording not found for {provider_name}.{tool_name} | Request: {kwargs}\n"
f"\n"
f"Run './scripts/integration-tests.sh --inference-mode record-if-missing' with required API keys to generate."
)
# If RECORD_IF_MISSING and no recording found, fall through to record
@ -644,10 +644,10 @@ async def _patched_inference_method(original_method, self, client_type, endpoint
elif mode == APIRecordingMode.REPLAY:
# REPLAY mode requires recording to exist
raise RuntimeError(
f"No recorded response found for request hash: {request_hash}\n"
f"Request: {method} {url} {body}\n"
f"Model: {body.get('model', 'unknown')}\n"
f"To record this response, run with LLAMA_STACK_TEST_INFERENCE_MODE=record"
f"Recording not found for request hash: {request_hash}\n"
f"Model: {body.get('model', 'unknown')} | Request: {method} {url}\n"
f"\n"
f"Run './scripts/integration-tests.sh --inference-mode record-if-missing' with required API keys to generate."
)
if mode == APIRecordingMode.RECORD or (mode == APIRecordingMode.RECORD_IF_MISSING and not recording):

2
tests/unit/distribution/test_api_recordings.py
View file

@ -261,7 +261,7 @@ class TestInferenceRecording:
with api_recording(mode=APIRecordingMode.REPLAY, storage_dir=str(temp_storage_dir)):
client = AsyncOpenAI(base_url="http://localhost:11434/v1", api_key="test")
with pytest.raises(RuntimeError, match="No recorded response found"):
with pytest.raises(RuntimeError, match="Recording not found"):
await client.chat.completions.create(
model="llama3.2:3b", messages=[{"role": "user", "content": "This was never recorded"}]
)