From 55067fa81df78ed7a28aa77e9b8d0676cd987f81 Mon Sep 17 00:00:00 2001 From: Sixian Yi Date: Sat, 18 Jan 2025 07:50:45 -0800 Subject: [PATCH] test report for v0.1 (#814) # What does this PR do? MD file for the test results of provider <> inference tests ## Test Plan 1) install `pip install pytest-md-report` 2) Run inference tests with the additions to the commands `--md-report --md-report-verbose=1 --md-report-output=tgi.md` Test text model: meta-llama/Llama-3.1-8B-Instruct Test vision model: meta-llama/Llama-3.2-11B-Vision-Instruct ## Sources Please link relevant resources if necessary. ## Before submitting - [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). - [ ] Ran pre-commit to handle lint / formatting issues. - [ ] Read the [contributor guideline](https://github.com/meta-llama/llama-stack/blob/main/CONTRIBUTING.md), Pull Request section? - [ ] Updated relevant documentation. - [ ] Wrote necessary unit or integration tests. --------- Co-authored-by: Xi Yan --- llama_stack/providers/tests/test_report.md | 70 ++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 llama_stack/providers/tests/test_report.md diff --git a/llama_stack/providers/tests/test_report.md b/llama_stack/providers/tests/test_report.md new file mode 100644 index 000000000..1153ef772 --- /dev/null +++ b/llama_stack/providers/tests/test_report.md @@ -0,0 +1,70 @@ +### Fireworks +| filepath | function | passed | SUBTOTAL | +| -------------------------------------------------------------- | ------------------------------------------------------------------ | -----: | -------: | +| llama_stack/providers/tests/inference/test_text_inference.py | TestInference.test_completion | 1 | 1 | +| llama_stack/providers/tests/inference/test_text_inference.py | TestInference.test_chat_completion_non_streaming | 1 | 1 | +| llama_stack/providers/tests/inference/test_text_inference.py | TestInference.test_structured_output | 1 | 1 | +| llama_stack/providers/tests/inference/test_text_inference.py | TestInference.test_chat_completion_streaming | 1 | 1 | +| llama_stack/providers/tests/inference/test_text_inference.py | TestInference.test_chat_completion_with_tool_calling | 1 | 1 | +| llama_stack/providers/tests/inference/test_text_inference.py | TestInference.test_chat_completion_with_tool_calling_streaming | 1 | 1 | +| llama_stack/providers/tests/inference/test_vision_inference.py | TestVisionModelInference.test_vision_chat_completion_non_streaming | 2 | 2 | +| llama_stack/providers/tests/inference/test_vision_inference.py | TestVisionModelInference.test_vision_chat_completion_streaming | 1 | 1 | +| TOTAL | | 9 | 9 | + + + +### Together +| filepath | function | passed | SUBTOTAL | +| -------------------------------------------------------------- | ------------------------------------------------------------------ | -----: | -------: | +| llama_stack/providers/tests/inference/test_text_inference.py | TestInference.test_completion | 1 | 1 | +| llama_stack/providers/tests/inference/test_text_inference.py | TestInference.test_chat_completion_non_streaming | 1 | 1 | +| llama_stack/providers/tests/inference/test_text_inference.py | TestInference.test_structured_output | 1 | 1 | +| llama_stack/providers/tests/inference/test_text_inference.py | TestInference.test_chat_completion_streaming | 1 | 1 | +| llama_stack/providers/tests/inference/test_text_inference.py | TestInference.test_chat_completion_with_tool_calling | 1 | 1 | +| llama_stack/providers/tests/inference/test_text_inference.py | TestInference.test_chat_completion_with_tool_calling_streaming | 1 | 1 | +| llama_stack/providers/tests/inference/test_vision_inference.py | TestVisionModelInference.test_vision_chat_completion_non_streaming | 2 | 2 | +| llama_stack/providers/tests/inference/test_vision_inference.py | TestVisionModelInference.test_vision_chat_completion_streaming | 1 | 1 | +| TOTAL | | 9 | 9 | + + +### vLLM + +| filepath | function | passed | skipped | SUBTOTAL | +| ------------------------------------------------------------ | -------------------------------------------------------------- | -----: | ------: | -------: | +| llama_stack/providers/tests/inference/test_text_inference.py | TestInference.test_model_list | 1 | 0 | 1 | +| llama_stack/providers/tests/inference/test_text_inference.py | TestInference.test_chat_completion_non_streaming | 1 | 0 | 1 | +| llama_stack/providers/tests/inference/test_text_inference.py | TestInference.test_structured_output | 1 | 0 | 1 | +| llama_stack/providers/tests/inference/test_text_inference.py | TestInference.test_chat_completion_streaming | 1 | 0 | 1 | +| llama_stack/providers/tests/inference/test_text_inference.py | TestInference.test_chat_completion_with_tool_calling | 1 | 0 | 1 | +| llama_stack/providers/tests/inference/test_text_inference.py | TestInference.test_chat_completion_with_tool_calling_streaming | 1 | 0 | 1 | +| llama_stack/providers/tests/inference/test_text_inference.py | TestInference.test_completion | 0 | 1 | 1 | +| llama_stack/providers/tests/inference/test_text_inference.py | TestInference.test_completion_logprobs | 0 | 1 | 1 | +| llama_stack/providers/tests/inference/test_text_inference.py | TestInference.test_completion_structured_output | 0 | 1 | 1 | +| TOTAL | | 6 | 3 | 9 | + +### Ollama +| filepath | function | passed | SUBTOTAL | +| ------------------------------------------------------------ | -------------------------------------------------------------- | -----: | -------: | +| llama_stack/providers/tests/inference/test_text_inference.py | TestInference.test_completion | 1 | 1 | +| llama_stack/providers/tests/inference/test_text_inference.py | TestInference.test_chat_completion_non_streaming | 1 | 1 | +| llama_stack/providers/tests/inference/test_text_inference.py | TestInference.test_structured_output | 1 | 1 | +| llama_stack/providers/tests/inference/test_text_inference.py | TestInference.test_chat_completion_streaming | 1 | 1 | +| llama_stack/providers/tests/inference/test_text_inference.py | TestInference.test_chat_completion_with_tool_calling | 1 | 1 | +| llama_stack/providers/tests/inference/test_text_inference.py | TestInference.test_chat_completion_with_tool_calling_streaming | 1 | 1 | +| TOTAL | | 6 | 6 | + + +### tgi + +| filepath | function | passed | skipped | SUBTOTAL | +| ------------------------------------------------ | -------------------------------------------------------------- | -----: | ------: | -------: | +| providers/tests/inference/test_text_inference.py | TestInference.test_model_list | 1 | 0 | 1 | +| providers/tests/inference/test_text_inference.py | TestInference.test_completion | 1 | 0 | 1 | +| providers/tests/inference/test_text_inference.py | TestInference.test_chat_completion_non_streaming | 1 | 0 | 1 | +| providers/tests/inference/test_text_inference.py | TestInference.test_structured_output | 1 | 0 | 1 | +| providers/tests/inference/test_text_inference.py | TestInference.test_chat_completion_streaming | 1 | 0 | 1 | +| providers/tests/inference/test_text_inference.py | TestInference.test_chat_completion_with_tool_calling | 1 | 0 | 1 | +| providers/tests/inference/test_text_inference.py | TestInference.test_chat_completion_with_tool_calling_streaming | 1 | 0 | 1 | +| providers/tests/inference/test_text_inference.py | TestInference.test_completion_logprobs | 0 | 1 | 1 | +| providers/tests/inference/test_text_inference.py | TestInference.test_completion_structured_output | 0 | 1 | 1 | +| TOTAL | | 7 | 2 | 9 |