eleuther eval provider

2025-12-09 11:20:58 +00:00 · 2024-10-04 13:45:52 -07:00 · 2024-10-04 13:45:52 -07:00 · 6234dd97d5
commit 6234dd97d5
parent 2441e66d14
7 changed files with 168 additions and 2 deletions
--- a/llama_stack/providers/registry/evals.py
+++ b/llama_stack/providers/registry/evals.py
@ -27,4 +27,16 @@ def available_providers() -> List[ProviderSpec]:
                Api.inference,
            ],
        ),
+        InlineProviderSpec(
+            api=Api.evals,
+            provider_type="eleuther",
+            pip_packages=[
+                "lm-eval",
+            ],
+            module="llama_stack.providers.adapters.evals.eleuther",
+            config_class="llama_stack.providers.adapters.evals.eleuther.EleutherEvalsImplConfig",
+            api_dependencies=[
+                Api.inference,
+            ],
+        ),
    ]