feat(verification): various improvements (#1921)

# What does this PR do?
- provider and their models now live in config.yaml
- better distinguish different cases within a test
- add model key to surface provider's model_id
- include example command to rerun single test case

## Test Plan
<img width="1173" alt="image"
src="https://github.com/user-attachments/assets/b414baf0-c768-451f-8c3b-c2905cf36fac"
/>
This commit is contained in:
ehhuang 2025-04-10 10:26:19 -07:00 committed by GitHub
parent 09a83b1ec1
commit 14146e4b3f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 4449 additions and 8810 deletions

View file

@ -0,0 +1,10 @@
base_url: https://api.cerebras.ai/v1
api_key_var: CEREBRAS_API_KEY
models:
- llama-3.3-70b
model_display_names:
llama-3.3-70b: Llama-3.3-70B-Instruct
test_exclusions:
llama-3.3-70b:
- test_chat_non_streaming_image
- test_chat_streaming_image

View file

@ -0,0 +1,14 @@
base_url: https://api.fireworks.ai/inference/v1
api_key_var: FIREWORKS_API_KEY
models:
- accounts/fireworks/models/llama-v3p3-70b-instruct
- accounts/fireworks/models/llama4-scout-instruct-basic
- accounts/fireworks/models/llama4-maverick-instruct-basic
model_display_names:
accounts/fireworks/models/llama-v3p3-70b-instruct: Llama-3.3-70B-Instruct
accounts/fireworks/models/llama4-scout-instruct-basic: Llama-4-Scout-Instruct
accounts/fireworks/models/llama4-maverick-instruct-basic: Llama-4-Maverick-Instruct
test_exclusions:
accounts/fireworks/models/llama-v3p3-70b-instruct:
- test_chat_non_streaming_image
- test_chat_streaming_image

View file

@ -0,0 +1,14 @@
base_url: https://api.groq.com/openai/v1
api_key_var: GROQ_API_KEY
models:
- llama-3.3-70b-versatile
- llama-4-scout-17b-16e-instruct
- llama-4-maverick-17b-128e-instruct
model_display_names:
llama-3.3-70b-versatile: Llama-3.3-70B-Instruct
llama-4-scout-17b-16e-instruct: Llama-4-Scout-Instruct
llama-4-maverick-17b-128e-instruct: Llama-4-Maverick-Instruct
test_exclusions:
llama-3.3-70b-versatile:
- test_chat_non_streaming_image
- test_chat_streaming_image

View file

@ -0,0 +1,9 @@
base_url: https://api.openai.com/v1
api_key_var: OPENAI_API_KEY
models:
- gpt-4o
- gpt-4o-mini
model_display_names:
gpt-4o: gpt-4o
gpt-4o-mini: gpt-4o-mini
test_exclusions: {}

View file

@ -0,0 +1,14 @@
base_url: https://api.together.xyz/v1
api_key_var: TOGETHER_API_KEY
models:
- meta-llama/Llama-3.3-70B-Instruct-Turbo
- meta-llama/Llama-4-Scout-17B-16E-Instruct
- meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8
model_display_names:
meta-llama/Llama-3.3-70B-Instruct-Turbo: Llama-3.3-70B-Instruct
meta-llama/Llama-4-Scout-17B-16E-Instruct: Llama-4-Scout-Instruct
meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8: Llama-4-Maverick-Instruct
test_exclusions:
meta-llama/Llama-3.3-70B-Instruct-Turbo:
- test_chat_non_streaming_image
- test_chat_streaming_image