diff --git a/docs/_static/llama-stack-spec.html b/docs/_static/llama-stack-spec.html
index 4fe542262..31e378711 100644
--- a/docs/_static/llama-stack-spec.html
+++ b/docs/_static/llama-stack-spec.html
@@ -6811,6 +6811,7 @@
                         "type": "string",
                         "enum": [
                             "post-training/messages",
+                            "eval/question-answer",
                             "eval/messages-answer"
                         ],
                         "title": "DatasetPurpose",
@@ -9885,9 +9886,10 @@
                         "type": "string",
                         "enum": [
                             "post-training/messages",
+                            "eval/question-answer",
                             "eval/messages-answer"
                         ],
-                        "description": "The purpose of the dataset. One of - \"post-training/messages\": The dataset contains a messages column with list of messages for post-training. - \"eval/messages-answer\": The dataset contains a messages column with list of messages and an answer column."
+                        "description": "The purpose of the dataset. One of - \"post-training/messages\": The dataset contains a messages column with list of messages for post-training. - \"eval/question-answer\": The dataset contains a question column and an answer column for evaluation. - \"eval/messages-answer\": The dataset contains a messages column with list of messages and an answer column for evaluation."
                     },
                     "source": {
                         "$ref": "#/components/schemas/DataSource",
diff --git a/docs/_static/llama-stack-spec.yaml b/docs/_static/llama-stack-spec.yaml
index d9939d1c0..ba5d3ab87 100644
--- a/docs/_static/llama-stack-spec.yaml
+++ b/docs/_static/llama-stack-spec.yaml
@@ -4716,6 +4716,7 @@ components:
           type: string
           enum:
             - post-training/messages
+            - eval/question-answer
             - eval/messages-answer
           title: DatasetPurpose
           description: >-
@@ -6776,12 +6777,14 @@ components:
           type: string
           enum:
             - post-training/messages
+            - eval/question-answer
             - eval/messages-answer
           description: >-
             The purpose of the dataset. One of - "post-training/messages": The dataset
             contains a messages column with list of messages for post-training. -
-            "eval/messages-answer": The dataset contains a messages column with list
-            of messages and an answer column.
+            "eval/question-answer": The dataset contains a question column and an
+            answer column for evaluation. - "eval/messages-answer": The dataset contains
+            a messages column with list of messages and an answer column for evaluation.
         source:
           $ref: '#/components/schemas/DataSource'
           description: >-
diff --git a/llama_stack/apis/datasets/datasets.py b/llama_stack/apis/datasets/datasets.py
index 9ec05a213..711e3289c 100644
--- a/llama_stack/apis/datasets/datasets.py
+++ b/llama_stack/apis/datasets/datasets.py
@@ -24,16 +24,23 @@ class DatasetPurpose(Enum):
                 {"role": "assistant", "content": "Hello, world!"},
             ]
         }
+    :cvar eval/question-answer: The dataset contains a question column and an answer column.
+        {
+            "question": "What is the capital of France?",
+            "answer": "Paris"
+        }
     :cvar eval/messages-answer: The dataset contains a messages column with list of messages and an answer column.
         {
             "messages": [
-                {"role": "user", "content": "What is the capital of France?"},
+                {"role": "user", "content": "Hello, my name is John Doe."},
+                {"role": "assistant", "content": "Hello, John Doe. How can I help you today?"},
+                {"role": "user", "content": "What's my name?"},
             ],
-            "answer": "Paris"
+            "answer": "John Doe"
         }
     """
-
     post_training_messages = "post-training/messages"
+    eval_question_answer = "eval/question-answer"
     eval_messages_answer = "eval/messages-answer"
 
     # TODO: add more schemas here
@@ -153,7 +160,8 @@ class Datasets(Protocol):
 
         :param purpose: The purpose of the dataset. One of
             - "post-training/messages": The dataset contains a messages column with list of messages for post-training.
-            - "eval/messages-answer": The dataset contains a messages column with list of messages and an answer column.
+            - "eval/question-answer": The dataset contains a question column and an answer column for evaluation.
+            - "eval/messages-answer": The dataset contains a messages column with list of messages and an answer column for evaluation.
         :param source: The data source of the dataset. Examples:
            - {
                "type": "uri",