eleuther generate until

llama_stack/providers/impls/meta_reference/inference/inference.py

@@ -91,10 +91,9 @@ class MetaReferenceInferenceImpl(Inference):
         else:
             return self._nonstream_chat_completion(request)
 
-    async def _nonstream_chat_completion(
+    def _nonstream_chat_completion(
         self, request: ChatCompletionRequest
     ) -> ChatCompletionResponse:
-        async with SEMAPHORE:
         messages = chat_completion_request_to_messages(request)
 
         tokens = []
@@ -121,18 +120,14 @@ class MetaReferenceInferenceImpl(Inference):
 
                 logprobs.append(
                     TokenLogProbs(
-                            logprobs_by_token={
-                                token_result.text: token_result.logprobs[0]
-                            }
+                        logprobs_by_token={token_result.text: token_result.logprobs[0]}
                     )
                 )
 
         if stop_reason is None:
             stop_reason = StopReason.out_of_tokens
 
-            message = self.generator.formatter.decode_assistant_message(
-                tokens, stop_reason
-            )
+        message = self.generator.formatter.decode_assistant_message(tokens, stop_reason)
         return ChatCompletionResponse(
             completion_message=message,
             logprobs=logprobs if request.logprobs else None,

llama_stack/providers/impls/third_party/evals/eleuther/eleuther.py

@@ -102,9 +102,8 @@ class EleutherEvalsWrapper(LM):
                 stream=False,
             )
             print(response)
-            res.append(response.completion_message)
+            res.append(response.completion_message.content)
 
-        print(response)
         return res
 
 

tests/examples/local-run.yaml

@@ -14,8 +14,8 @@ apis:
 - evals
 providers:
   evals:
-  - provider_id: meta-reference
-    provider_type: meta-reference
+  - provider_id: eleuther
+    provider_type: eleuther
     config: {}
   inference:
   - provider_id: meta-reference