Merge branch 'main' into playground-ui

llama_stack/providers/inline/scoring/__init__.py

@@ -0,0 +1,5 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.

llama_stack/providers/inline/scoring/braintrust/__init__.py

@@ -5,9 +5,10 @@
 # the root directory of this source tree.
 from typing import Dict
 
-from llama_stack.distribution.datatypes import Api, ProviderSpec
 from pydantic import BaseModel
 
+from llama_stack.distribution.datatypes import Api, ProviderSpec
+
 from .config import BraintrustScoringConfig
 
 

llama_stack/providers/inline/scoring/braintrust/braintrust.py

@@ -16,6 +16,7 @@ import os
 
 from autoevals.llm import Factuality
 from autoevals.ragas import AnswerCorrectness
+
 from llama_stack.distribution.request_headers import NeedsRequestProviderData
 from llama_stack.providers.datatypes import ScoringFunctionsProtocolPrivate
 

llama_stack/providers/tests/inference/test_text_inference.py

@@ -211,7 +211,15 @@ class TestInference:
         response = await inference_impl.chat_completion(
             model_id=inference_model,
             messages=[
-                SystemMessage(content="You are a helpful assistant."),
+                # we include context about Michael Jordan in the prompt so that the test is
+                # focused on the funtionality of the model and not on the information embedded
+                # in the model. Llama 3.2 3B Instruct tends to think MJ played for 14 seasons.
+                SystemMessage(
+                    content=(
+                        "You are a helpful assistant.\n\n"
+                        "Michael Jordan was born in 1963. He played basketball for the Chicago Bulls for 15 seasons."
+                    )
+                ),
                 UserMessage(content="Please give me information about Michael Jordan."),
             ],
             stream=False,