mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-08-21 17:33:12 +00:00
feat(ci): make recording workflow simpler, more parameterizable (#3169)
Some checks failed
SqlStore Integration Tests / test-postgres (3.12) (push) Failing after 1s
Test External Providers Installed via Module / test-external-providers-from-module (venv) (push) Has been skipped
Python Package Build Test / build (3.13) (push) Failing after 4s
Integration Tests (Replay) / Integration Tests (, , , client=, vision=) (push) Failing after 7s
Python Package Build Test / build (3.12) (push) Failing after 12s
Integration Auth Tests / test-matrix (oauth2_token) (push) Failing after 14s
Update ReadTheDocs / update-readthedocs (push) Failing after 12s
SqlStore Integration Tests / test-postgres (3.13) (push) Failing after 17s
Test External API and Providers / test-external (venv) (push) Failing after 15s
Vector IO Integration Tests / test-matrix (push) Failing after 28s
Unit Tests / unit-tests (3.12) (push) Failing after 27s
Unit Tests / unit-tests (3.13) (push) Failing after 51s
Pre-commit / pre-commit (push) Successful in 2m6s
Some checks failed
SqlStore Integration Tests / test-postgres (3.12) (push) Failing after 1s
Test External Providers Installed via Module / test-external-providers-from-module (venv) (push) Has been skipped
Python Package Build Test / build (3.13) (push) Failing after 4s
Integration Tests (Replay) / Integration Tests (, , , client=, vision=) (push) Failing after 7s
Python Package Build Test / build (3.12) (push) Failing after 12s
Integration Auth Tests / test-matrix (oauth2_token) (push) Failing after 14s
Update ReadTheDocs / update-readthedocs (push) Failing after 12s
SqlStore Integration Tests / test-postgres (3.13) (push) Failing after 17s
Test External API and Providers / test-external (venv) (push) Failing after 15s
Vector IO Integration Tests / test-matrix (push) Failing after 28s
Unit Tests / unit-tests (3.12) (push) Failing after 27s
Unit Tests / unit-tests (3.13) (push) Failing after 51s
Pre-commit / pre-commit (push) Successful in 2m6s
# What does this PR do? Recording tests has become a nightmare. This is the first part of making that process simpler by making it _less_ automatic. I tried to be too clever earlier. It simplifies the record-integration-tests workflow to use workflow dispatch inputs instead of PR labels. No more opaque stuff. Just go to the GitHub UI and run the workflow with inputs. I will soon add a helper script for this also. Other things to aid re-running just the small set of things you need to re-record: - Replaces the `test-types` JSON array parameter with a more intuitive `test-subdirs` comma-separated list. The whole JSON array crap was for matrix. - Adds a new `test-pattern` parameter to allow filtering tests using pytest's `-k` option ## Test Plan Note that this PR is in a fork not the source repository. - Replay tests on this PR are green - Manually [ran](1699856292) the replay workflow with a test-subdir and test-pattern filter, worked - Manually [ran](4819508034) the **record** workflow with a simple pattern, it has worked and updated _this_ PR. --------- Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
This commit is contained in:
Ashwin Bharambe
GitHub
parent
a6e2c18909
commit
0e8bb94bf3
7 changed files with 119 additions and 221 deletions
|
|
@ -14,7 +14,7 @@
|
|||
"models": [
|
||||
{
|
||||
"model": "nomic-embed-text:latest",
|
||||
"modified_at": "2025-08-14T20:26:10.795125-07:00",
|
||||
"modified_at": "2025-08-15T20:24:13.254634Z",
|
||||
"digest": "0a109f422b47e3a30ba2b10eca18548e944e8a23073ee3f3e947efcf3c45e59f",
|
||||
"size": 274302450,
|
||||
"details": {
|
||||
|
|
@ -28,41 +28,9 @@
|
|||
"quantization_level": "F16"
|
||||
}
|
||||
},
|
||||
{
|
||||
"model": "llama3.2-vision:11b",
|
||||
"modified_at": "2025-07-30T18:45:02.517873-07:00",
|
||||
"digest": "6f2f9757ae97e8a3f8ea33d6adb2b11d93d9a35bef277cd2c0b1b5af8e8d0b1e",
|
||||
"size": 7816589186,
|
||||
"details": {
|
||||
"parent_model": "",
|
||||
"format": "gguf",
|
||||
"family": "mllama",
|
||||
"families": [
|
||||
"mllama"
|
||||
],
|
||||
"parameter_size": "10.7B",
|
||||
"quantization_level": "Q4_K_M"
|
||||
}
|
||||
},
|
||||
{
|
||||
"model": "llama3.2-vision:latest",
|
||||
"modified_at": "2025-07-29T20:18:47.920468-07:00",
|
||||
"digest": "6f2f9757ae97e8a3f8ea33d6adb2b11d93d9a35bef277cd2c0b1b5af8e8d0b1e",
|
||||
"size": 7816589186,
|
||||
"details": {
|
||||
"parent_model": "",
|
||||
"format": "gguf",
|
||||
"family": "mllama",
|
||||
"families": [
|
||||
"mllama"
|
||||
],
|
||||
"parameter_size": "10.7B",
|
||||
"quantization_level": "Q4_K_M"
|
||||
}
|
||||
},
|
||||
{
|
||||
"model": "llama-guard3:1b",
|
||||
"modified_at": "2025-07-25T14:39:44.978630-07:00",
|
||||
"modified_at": "2025-07-31T04:44:58Z",
|
||||
"digest": "494147e06bf99e10dbe67b63a07ac81c162f18ef3341aa3390007ac828571b3b",
|
||||
"size": 1600181919,
|
||||
"details": {
|
||||
|
|
@ -78,7 +46,7 @@
|
|||
},
|
||||
{
|
||||
"model": "all-minilm:l6-v2",
|
||||
"modified_at": "2025-07-24T15:15:11.129290-07:00",
|
||||
"modified_at": "2025-07-31T04:42:15Z",
|
||||
"digest": "1b226e2802dbb772b5fc32a58f103ca1804ef7501331012de126ab22f67475ef",
|
||||
"size": 45960996,
|
||||
"details": {
|
||||
|
|
@ -92,57 +60,9 @@
|
|||
"quantization_level": "F16"
|
||||
}
|
||||
},
|
||||
{
|
||||
"model": "llama3.2:1b",
|
||||
"modified_at": "2025-07-17T22:02:24.953208-07:00",
|
||||
"digest": "baf6a787fdffd633537aa2eb51cfd54cb93ff08e28040095462bb63daf552878",
|
||||
"size": 1321098329,
|
||||
"details": {
|
||||
"parent_model": "",
|
||||
"format": "gguf",
|
||||
"family": "llama",
|
||||
"families": [
|
||||
"llama"
|
||||
],
|
||||
"parameter_size": "1.2B",
|
||||
"quantization_level": "Q8_0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"model": "all-minilm:latest",
|
||||
"modified_at": "2025-06-03T16:50:10.946583-07:00",
|
||||
"digest": "1b226e2802dbb772b5fc32a58f103ca1804ef7501331012de126ab22f67475ef",
|
||||
"size": 45960996,
|
||||
"details": {
|
||||
"parent_model": "",
|
||||
"format": "gguf",
|
||||
"family": "bert",
|
||||
"families": [
|
||||
"bert"
|
||||
],
|
||||
"parameter_size": "23M",
|
||||
"quantization_level": "F16"
|
||||
}
|
||||
},
|
||||
{
|
||||
"model": "llama3.2:3b",
|
||||
"modified_at": "2025-05-01T11:15:23.797447-07:00",
|
||||
"digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72",
|
||||
"size": 2019393189,
|
||||
"details": {
|
||||
"parent_model": "",
|
||||
"format": "gguf",
|
||||
"family": "llama",
|
||||
"families": [
|
||||
"llama"
|
||||
],
|
||||
"parameter_size": "3.2B",
|
||||
"quantization_level": "Q4_K_M"
|
||||
}
|
||||
},
|
||||
{
|
||||
"model": "llama3.2:3b-instruct-fp16",
|
||||
"modified_at": "2025-04-30T15:33:48.939665-07:00",
|
||||
"modified_at": "2025-07-31T04:42:05Z",
|
||||
"digest": "195a8c01d91ec3cb1e0aad4624a51f2602c51fa7d96110f8ab5a20c84081804d",
|
||||
"size": 6433703586,
|
||||
"details": {
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@
|
|||
"__type__": "ollama._types.GenerateResponse",
|
||||
"__data__": {
|
||||
"model": "llama3.2:3b-instruct-fp16",
|
||||
"created_at": "2025-08-04T22:55:14.141947Z",
|
||||
"created_at": "2025-08-15T20:24:49.18651486Z",
|
||||
"done": false,
|
||||
"done_reason": null,
|
||||
"total_duration": null,
|
||||
|
|
@ -39,7 +39,7 @@
|
|||
"__type__": "ollama._types.GenerateResponse",
|
||||
"__data__": {
|
||||
"model": "llama3.2:3b-instruct-fp16",
|
||||
"created_at": "2025-08-04T22:55:14.194979Z",
|
||||
"created_at": "2025-08-15T20:24:49.370611348Z",
|
||||
"done": false,
|
||||
"done_reason": null,
|
||||
"total_duration": null,
|
||||
|
|
@ -57,7 +57,7 @@
|
|||
"__type__": "ollama._types.GenerateResponse",
|
||||
"__data__": {
|
||||
"model": "llama3.2:3b-instruct-fp16",
|
||||
"created_at": "2025-08-04T22:55:14.248312Z",
|
||||
"created_at": "2025-08-15T20:24:49.557000029Z",
|
||||
"done": false,
|
||||
"done_reason": null,
|
||||
"total_duration": null,
|
||||
|
|
@ -75,7 +75,7 @@
|
|||
"__type__": "ollama._types.GenerateResponse",
|
||||
"__data__": {
|
||||
"model": "llama3.2:3b-instruct-fp16",
|
||||
"created_at": "2025-08-04T22:55:14.301911Z",
|
||||
"created_at": "2025-08-15T20:24:49.746777116Z",
|
||||
"done": false,
|
||||
"done_reason": null,
|
||||
"total_duration": null,
|
||||
|
|
@ -93,7 +93,7 @@
|
|||
"__type__": "ollama._types.GenerateResponse",
|
||||
"__data__": {
|
||||
"model": "llama3.2:3b-instruct-fp16",
|
||||
"created_at": "2025-08-04T22:55:14.354437Z",
|
||||
"created_at": "2025-08-15T20:24:49.942233333Z",
|
||||
"done": false,
|
||||
"done_reason": null,
|
||||
"total_duration": null,
|
||||
|
|
@ -111,7 +111,7 @@
|
|||
"__type__": "ollama._types.GenerateResponse",
|
||||
"__data__": {
|
||||
"model": "llama3.2:3b-instruct-fp16",
|
||||
"created_at": "2025-08-04T22:55:14.406821Z",
|
||||
"created_at": "2025-08-15T20:24:50.126788846Z",
|
||||
"done": false,
|
||||
"done_reason": null,
|
||||
"total_duration": null,
|
||||
|
|
@ -129,7 +129,7 @@
|
|||
"__type__": "ollama._types.GenerateResponse",
|
||||
"__data__": {
|
||||
"model": "llama3.2:3b-instruct-fp16",
|
||||
"created_at": "2025-08-04T22:55:14.457633Z",
|
||||
"created_at": "2025-08-15T20:24:50.311346131Z",
|
||||
"done": false,
|
||||
"done_reason": null,
|
||||
"total_duration": null,
|
||||
|
|
@ -147,7 +147,7 @@
|
|||
"__type__": "ollama._types.GenerateResponse",
|
||||
"__data__": {
|
||||
"model": "llama3.2:3b-instruct-fp16",
|
||||
"created_at": "2025-08-04T22:55:14.507857Z",
|
||||
"created_at": "2025-08-15T20:24:50.501507173Z",
|
||||
"done": false,
|
||||
"done_reason": null,
|
||||
"total_duration": null,
|
||||
|
|
@ -165,7 +165,7 @@
|
|||
"__type__": "ollama._types.GenerateResponse",
|
||||
"__data__": {
|
||||
"model": "llama3.2:3b-instruct-fp16",
|
||||
"created_at": "2025-08-04T22:55:14.558847Z",
|
||||
"created_at": "2025-08-15T20:24:50.692296777Z",
|
||||
"done": false,
|
||||
"done_reason": null,
|
||||
"total_duration": null,
|
||||
|
|
@ -183,7 +183,7 @@
|
|||
"__type__": "ollama._types.GenerateResponse",
|
||||
"__data__": {
|
||||
"model": "llama3.2:3b-instruct-fp16",
|
||||
"created_at": "2025-08-04T22:55:14.609969Z",
|
||||
"created_at": "2025-08-15T20:24:50.878846539Z",
|
||||
"done": false,
|
||||
"done_reason": null,
|
||||
"total_duration": null,
|
||||
|
|
@ -201,15 +201,15 @@
|
|||
"__type__": "ollama._types.GenerateResponse",
|
||||
"__data__": {
|
||||
"model": "llama3.2:3b-instruct-fp16",
|
||||
"created_at": "2025-08-04T22:55:14.660997Z",
|
||||
"created_at": "2025-08-15T20:24:51.063200561Z",
|
||||
"done": true,
|
||||
"done_reason": "stop",
|
||||
"total_duration": 715356542,
|
||||
"load_duration": 59747500,
|
||||
"total_duration": 33982453650,
|
||||
"load_duration": 2909001805,
|
||||
"prompt_eval_count": 341,
|
||||
"prompt_eval_duration": 128000000,
|
||||
"prompt_eval_duration": 29194357307,
|
||||
"eval_count": 11,
|
||||
"eval_duration": 526000000,
|
||||
"eval_duration": 1878247732,
|
||||
"response": "",
|
||||
"thinking": null,
|
||||
"context": null
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue