Merge branch 'main' into sambanova-inferene

llama_stack/distribution/ui/README.md

@@ -1,16 +1,41 @@
-# LLama Stack UI
+# (Experimental) LLama Stack UI
 
-[!NOTE] This is a work in progress.
+## Docker Setup
 
-## Prerequisite
-- Start up Llama Stack Server
-```
-llama stack run
-```
+:warning: This is a work in progress.
 
-## Running Streamlit App
+## Developer Setup
+
+1. Start up Llama Stack API server. More details [here](https://llama-stack.readthedocs.io/en/latest/getting_started/index.html).
 
 ```
+llama stack build --template together --image-type conda
+
+llama stack run together
+```
+
+2. (Optional) Register datasets and eval tasks as resources. If you want to run pre-configured evaluation flows (e.g. Evaluations (Generation + Scoring) Page).
+
+```bash
+$ llama-stack-client datasets register \
+--dataset-id "mmlu" \
+--provider-id "huggingface" \
+--url "https://huggingface.co/datasets/llamastack/evals" \
+--metadata '{"path": "llamastack/evals", "name": "evals__mmlu__details", "split": "train"}' \
+--schema '{"input_query": {"type": "string"}, "expected_answer": {"type": "string", "chat_completion_input": {"type": "string"}}}'
+```
+
+```bash
+$ llama-stack-client eval_tasks register \
+--eval-task-id meta-reference-mmlu \
+--provider-id meta-reference \
+--dataset-id mmlu \
+--scoring-functions basic::regex_parser_multiple_choice_answer
+```
+
+3. Start Streamlit UI
+
+```bash
 cd llama_stack/distribution/ui
 pip install -r requirements.txt
 streamlit run app.py

llama_stack/distribution/ui/page/evaluations/app_eval.py

@@ -129,7 +129,7 @@ def application_evaluation_page():
 
                 # Display current row results using separate containers
                 progress_text_container.write(
-                    f"Expand to see current processed result ({i+1}/{len(rows)})"
+                    f"Expand to see current processed result ({i + 1} / {len(rows)})"
                 )
                 results_container.json(
                     score_res.to_json(),

llama_stack/distribution/ui/page/evaluations/native_eval.py

@@ -232,7 +232,7 @@ def run_evaluation_3():
                 output_res[scoring_fn].append(eval_res.scores[scoring_fn].score_rows[0])
 
             progress_text_container.write(
-                f"Expand to see current processed result ({i+1}/{len(rows)})"
+                f"Expand to see current processed result ({i + 1} / {len(rows)})"
             )
             results_container.json(eval_res, expanded=2)
 

llama_stack/distribution/ui/page/playground/chat.py

@@ -11,7 +11,9 @@ from modules.api import llama_stack_api
 with st.sidebar:
     st.header("Configuration")
     available_models = llama_stack_api.client.models.list()
-    available_models = [model.identifier for model in available_models]
+    available_models = [
+        model.identifier for model in available_models if model.model_type == "llm"
+    ]
     selected_model = st.selectbox(
         "Choose a model",
         available_models,

llama_stack/distribution/ui/page/playground/rag.py

@@ -74,7 +74,9 @@ def rag_chat_page():
         ]
 
         available_models = llama_stack_api.client.models.list()
-        available_models = [model.identifier for model in available_models]
+        available_models = [
+            model.identifier for model in available_models if model.model_type == "llm"
+        ]
         selected_model = st.selectbox(
             "Choose a model",
             available_models,
@@ -116,8 +118,6 @@ def rag_chat_page():
         with st.chat_message(message["role"]):
             st.markdown(message["content"])
 
-    selected_model = llama_stack_api.client.models.list()[0].identifier
-
     agent_config = AgentConfig(
         model=selected_model,
         instructions=system_prompt,