From 97f435b4df7a082244f5c4f76da163bb47d3e9c7 Mon Sep 17 00:00:00 2001 From: Sixian Yi Date: Wed, 22 Jan 2025 18:27:42 -0800 Subject: [PATCH] fix --- llama_stack/templates/fireworks/report.md | 28 +++---- llama_stack/templates/ollama/report.md | 40 +++++----- llama_stack/templates/together/report.md | 28 +++---- tests/client-sdk/report.py | 95 ++++++++++++++++++++--- 4 files changed, 131 insertions(+), 60 deletions(-) diff --git a/llama_stack/templates/fireworks/report.md b/llama_stack/templates/fireworks/report.md index 9306d327c..1c5550bf4 100644 --- a/llama_stack/templates/fireworks/report.md +++ b/llama_stack/templates/fireworks/report.md @@ -3,20 +3,20 @@ ## Supported Models: | Model Descriptor | fireworks | |:---|:---| -| meta-llama/Llama-3-8B-Instruct | ❌ | -| meta-llama/Llama-3-70B-Instruct | ❌ | -| meta-llama/Llama-3.1-8B-Instruct | ✅ | -| meta-llama/Llama-3.1-70B-Instruct | ✅ | -| meta-llama/Llama-3.1-405B-Instruct-FP8 | ✅ | -| meta-llama/Llama-3.2-1B-Instruct | ✅ | -| meta-llama/Llama-3.2-3B-Instruct | ✅ | -| meta-llama/Llama-3.2-11B-Vision-Instruct | ✅ | -| meta-llama/Llama-3.2-90B-Vision-Instruct | ✅ | -| meta-llama/Llama-3.3-70B-Instruct | ✅ | -| meta-llama/Llama-Guard-3-11B-Vision | ✅ | -| meta-llama/Llama-Guard-3-1B | ❌ | -| meta-llama/Llama-Guard-3-8B | ✅ | -| meta-llama/Llama-Guard-2-8B | ❌ | +| Llama-3-8B-Instruct | ❌ | +| Llama-3-70B-Instruct | ❌ | +| Llama3.1-8B-Instruct | ✅ | +| Llama3.1-70B-Instruct | ✅ | +| Llama3.1-405B-Instruct | ✅ | +| Llama3.2-1B-Instruct | ✅ | +| Llama3.2-3B-Instruct | ✅ | +| Llama3.2-11B-Vision-Instruct | ✅ | +| Llama3.2-90B-Vision-Instruct | ✅ | +| Llama3.3-70B-Instruct | ✅ | +| Llama-Guard-3-11B-Vision | ✅ | +| Llama-Guard-3-1B | ❌ | +| Llama-Guard-3-8B | ✅ | +| Llama-Guard-2-8B | ❌ | ## Inference: | Model | API | Capability | Test | Status | diff --git a/llama_stack/templates/ollama/report.md b/llama_stack/templates/ollama/report.md index a433d317c..0d370b8ec 100644 --- a/llama_stack/templates/ollama/report.md +++ b/llama_stack/templates/ollama/report.md @@ -3,27 +3,27 @@ ## Supported Models: | Model Descriptor | ollama | |:---|:---| -| meta-llama/Llama-3-8B-Instruct | ❌ | -| meta-llama/Llama-3-70B-Instruct | ❌ | -| meta-llama/Llama-3.1-8B-Instruct | ✅ | -| meta-llama/Llama-3.1-70B-Instruct | ❌ | -| meta-llama/Llama-3.1-405B-Instruct-FP8 | ❌ | -| meta-llama/Llama-3.2-1B-Instruct | ❌ | -| meta-llama/Llama-3.2-3B-Instruct | ❌ | -| meta-llama/Llama-3.2-11B-Vision-Instruct | ❌ | -| meta-llama/Llama-3.2-90B-Vision-Instruct | ❌ | -| meta-llama/Llama-3.3-70B-Instruct | ❌ | -| meta-llama/Llama-Guard-3-11B-Vision | ❌ | -| meta-llama/Llama-Guard-3-1B | ❌ | -| meta-llama/Llama-Guard-3-8B | ❌ | -| meta-llama/Llama-Guard-2-8B | ❌ | +| Llama-3-8B-Instruct | ❌ | +| Llama-3-70B-Instruct | ❌ | +| Llama3.1-8B-Instruct | ✅ | +| Llama3.1-70B-Instruct | ✅ | +| Llama3.1-405B-Instruct | ✅ | +| Llama3.2-1B-Instruct | ✅ | +| Llama3.2-3B-Instruct | ✅ | +| Llama3.2-11B-Vision-Instruct | ✅ | +| Llama3.2-90B-Vision-Instruct | ✅ | +| Llama3.3-70B-Instruct | ✅ | +| Llama-Guard-3-11B-Vision | ❌ | +| Llama-Guard-3-1B | ✅ | +| Llama-Guard-3-8B | ✅ | +| Llama-Guard-2-8B | ❌ | ## Inference: | Model | API | Capability | Test | Status | |:----- |:-----|:-----|:-----|:-----| | Llama-3.1-8B-Instruct | /chat_completion | streaming | test_text_chat_completion_streaming | ✅ | -| Llama-3.1-8B-Instruct | /chat_completion | streaming | test_image_chat_completion_streaming | ❌ | -| Llama-3.1-8B-Instruct | /chat_completion | non_streaming | test_image_chat_completion_non_streaming | ❌ | +| Llama-3.2-11B-Vision-Instruct | /chat_completion | streaming | test_image_chat_completion_streaming | ❌ | +| Llama-3.2-11B-Vision-Instruct | /chat_completion | non_streaming | test_image_chat_completion_non_streaming | ❌ | | Llama-3.1-8B-Instruct | /chat_completion | non_streaming | test_text_chat_completion_non_streaming | ✅ | | Llama-3.1-8B-Instruct | /chat_completion | tool_calling | test_text_chat_completion_with_tool_calling_and_streaming | ✅ | | Llama-3.1-8B-Instruct | /chat_completion | tool_calling | test_text_chat_completion_with_tool_calling_and_non_streaming | ✅ | @@ -34,11 +34,11 @@ ## Vector_io: | API | Capability | Test | Status | |:-----|:-----|:-----|:-----| -| retrieve | | test_vector_db_retrieve | ✅ | +| /retrieve | | test_vector_db_retrieve | ✅ | ## Agents: | API | Capability | Test | Status | |:-----|:-----|:-----|:-----| -| create_agent_turn | rag | test_rag_agent | ✅ | -| create_agent_turn | custom_tool | test_custom_tool | ❌ | -| create_agent_turn | code_execution | test_code_interpreter_for_attachments | ✅ | +| /create_agent_turn | rag | test_rag_agent | ✅ | +| /create_agent_turn | custom_tool | test_custom_tool | ✅ | +| /create_agent_turn | code_execution | test_code_interpreter_for_attachments | ✅ | diff --git a/llama_stack/templates/together/report.md b/llama_stack/templates/together/report.md index 294088bf0..10891f4e5 100644 --- a/llama_stack/templates/together/report.md +++ b/llama_stack/templates/together/report.md @@ -3,20 +3,20 @@ ## Supported Models: | Model Descriptor | together | |:---|:---| -| meta-llama/Llama-3-8B-Instruct | ❌ | -| meta-llama/Llama-3-70B-Instruct | ❌ | -| meta-llama/Llama-3.1-8B-Instruct | ✅ | -| meta-llama/Llama-3.1-70B-Instruct | ✅ | -| meta-llama/Llama-3.1-405B-Instruct-FP8 | ✅ | -| meta-llama/Llama-3.2-1B-Instruct | ❌ | -| meta-llama/Llama-3.2-3B-Instruct | ✅ | -| meta-llama/Llama-3.2-11B-Vision-Instruct | ✅ | -| meta-llama/Llama-3.2-90B-Vision-Instruct | ✅ | -| meta-llama/Llama-3.3-70B-Instruct | ✅ | -| meta-llama/Llama-Guard-3-11B-Vision | ✅ | -| meta-llama/Llama-Guard-3-1B | ❌ | -| meta-llama/Llama-Guard-3-8B | ✅ | -| meta-llama/Llama-Guard-2-8B | ❌ | +| Llama-3-8B-Instruct | ❌ | +| Llama-3-70B-Instruct | ❌ | +| Llama3.1-8B-Instruct | ✅ | +| Llama3.1-70B-Instruct | ✅ | +| Llama3.1-405B-Instruct | ✅ | +| Llama3.2-1B-Instruct | ❌ | +| Llama3.2-3B-Instruct | ✅ | +| Llama3.2-11B-Vision-Instruct | ✅ | +| Llama3.2-90B-Vision-Instruct | ✅ | +| Llama3.3-70B-Instruct | ✅ | +| Llama-Guard-3-11B-Vision | ✅ | +| Llama-Guard-3-1B | ❌ | +| Llama-Guard-3-8B | ✅ | +| Llama-Guard-2-8B | ❌ | ## Inference: | Model | API | Capability | Test | Status | diff --git a/tests/client-sdk/report.py b/tests/client-sdk/report.py index ba85d004f..8d295d87f 100644 --- a/tests/client-sdk/report.py +++ b/tests/client-sdk/report.py @@ -12,8 +12,9 @@ from pathlib import Path from urllib.parse import urlparse import pytest - +from llama_models.datatypes import CoreModelId from llama_models.sku_list import ( + all_registered_models, llama3_1_instruct_models, llama3_2_instruct_models, llama3_3_instruct_models, @@ -40,7 +41,79 @@ def featured_models_repo_names(): *llama3_3_instruct_models(), *safety_models(), ] - return [model.huggingface_repo for model in models if not model.variant] + return [model for model in models if not model.variant] + + +SUPPORTED_MODELS = { + "ollama": set( + [ + CoreModelId.llama3_1_8b_instruct.value, + CoreModelId.llama3_1_8b_instruct.value, + CoreModelId.llama3_1_70b_instruct.value, + CoreModelId.llama3_1_70b_instruct.value, + CoreModelId.llama3_1_405b_instruct.value, + CoreModelId.llama3_1_405b_instruct.value, + CoreModelId.llama3_2_1b_instruct.value, + CoreModelId.llama3_2_1b_instruct.value, + CoreModelId.llama3_2_3b_instruct.value, + CoreModelId.llama3_2_3b_instruct.value, + CoreModelId.llama3_2_11b_vision_instruct.value, + CoreModelId.llama3_2_11b_vision_instruct.value, + CoreModelId.llama3_2_90b_vision_instruct.value, + CoreModelId.llama3_2_90b_vision_instruct.value, + CoreModelId.llama3_3_70b_instruct.value, + CoreModelId.llama_guard_3_8b.value, + CoreModelId.llama_guard_3_1b.value, + ] + ), + "fireworks": set( + [ + CoreModelId.llama3_1_8b_instruct.value, + CoreModelId.llama3_1_70b_instruct.value, + CoreModelId.llama3_1_405b_instruct.value, + CoreModelId.llama3_2_1b_instruct.value, + CoreModelId.llama3_2_3b_instruct.value, + CoreModelId.llama3_2_11b_vision_instruct.value, + CoreModelId.llama3_2_90b_vision_instruct.value, + CoreModelId.llama3_3_70b_instruct.value, + CoreModelId.llama_guard_3_8b.value, + CoreModelId.llama_guard_3_11b_vision.value, + ] + ), + "together": set( + [ + CoreModelId.llama3_1_8b_instruct.value, + CoreModelId.llama3_1_70b_instruct.value, + CoreModelId.llama3_1_405b_instruct.value, + CoreModelId.llama3_2_3b_instruct.value, + CoreModelId.llama3_2_11b_vision_instruct.value, + CoreModelId.llama3_2_90b_vision_instruct.value, + CoreModelId.llama3_3_70b_instruct.value, + CoreModelId.llama_guard_3_8b.value, + CoreModelId.llama_guard_3_11b_vision.value, + ] + ), + "tgi": set( + [ + model.core_model_id.value + for model in all_registered_models() + if model.huggingface_repo + ] + ), + "vllm": set( + [ + model.core_model_id.value + for model in all_registered_models() + if model.huggingface_repo + ] + ), + "cerebras": set( + [ + CoreModelId.llama3_1_8b_instruct.value, + CoreModelId.llama3_3_70b_instruct.value, + ] + ), +} class Report: @@ -119,16 +192,14 @@ class Report: report.append(dividor) rows = [] - - try: - supported_models = {m.identifier for m in self.client.models.list()} - except Exception as e: - cprint(f"Error getting models: {e}", "red") - supported_models = set() - - for m_name in featured_models_repo_names(): - row = f"| {m_name} |" - if m_name in supported_models: + for model in all_registered_models(): + if ( + "Instruct" not in model.core_model_id.value + and "Guard" not in model.core_model_id.value + ) or (model.variant): + continue + row = f"| {model.core_model_id.value} |" + if model.core_model_id.value in SUPPORTED_MODELS[self.image_name]: row += " ✅ |" else: row += " ❌ |"