version: 2 image_name: open-benchmark apis: - agents - datasetio - eval - inference - safety - scoring - tool_runtime - vector_io providers: inference: - provider_id: openai provider_type: remote::openai config: api_key: ${env.OPENAI_API_KEY:=} base_url: ${env.OPENAI_BASE_URL:=https://api.openai.com/v1} - provider_id: anthropic provider_type: remote::anthropic config: api_key: ${env.ANTHROPIC_API_KEY:=} - provider_id: gemini provider_type: remote::gemini config: api_key: ${env.GEMINI_API_KEY:=} - provider_id: groq provider_type: remote::groq config: url: https://api.groq.com api_key: ${env.GROQ_API_KEY:=} - provider_id: together provider_type: remote::together config: url: https://api.together.xyz/v1 api_key: ${env.TOGETHER_API_KEY:=} vector_io: - provider_id: sqlite-vec provider_type: inline::sqlite-vec config: db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/open-benchmark}/sqlite_vec.db persistence: namespace: vector_io::sqlite_vec backend: kv_default - provider_id: ${env.ENABLE_CHROMADB:+chromadb} provider_type: remote::chromadb config: url: ${env.CHROMADB_URL:=} persistence: namespace: vector_io::chroma_remote backend: kv_default - provider_id: ${env.ENABLE_PGVECTOR:+pgvector} provider_type: remote::pgvector config: host: ${env.PGVECTOR_HOST:=localhost} port: ${env.PGVECTOR_PORT:=5432} db: ${env.PGVECTOR_DB:=} user: ${env.PGVECTOR_USER:=} password: ${env.PGVECTOR_PASSWORD:=} persistence: namespace: vector_io::pgvector backend: kv_default safety: - provider_id: llama-guard provider_type: inline::llama-guard config: excluded_categories: [] agents: - provider_id: meta-reference provider_type: inline::meta-reference config: persistence: agent_state: namespace: agents backend: kv_default responses: table_name: responses backend: sql_default max_write_queue_size: 10000 num_writers: 4 eval: - provider_id: meta-reference provider_type: inline::meta-reference config: kvstore: namespace: eval backend: kv_default datasetio: - provider_id: huggingface provider_type: remote::huggingface config: kvstore: namespace: datasetio::huggingface backend: kv_default - provider_id: localfs provider_type: inline::localfs config: kvstore: namespace: datasetio::localfs backend: kv_default scoring: - provider_id: basic provider_type: inline::basic - provider_id: llm-as-judge provider_type: inline::llm-as-judge - provider_id: braintrust provider_type: inline::braintrust config: openai_api_key: ${env.OPENAI_API_KEY:=} tool_runtime: - provider_id: brave-search provider_type: remote::brave-search config: api_key: ${env.BRAVE_SEARCH_API_KEY:=} max_results: 3 - provider_id: tavily-search provider_type: remote::tavily-search config: api_key: ${env.TAVILY_SEARCH_API_KEY:=} max_results: 3 - provider_id: model-context-protocol provider_type: remote::model-context-protocol storage: backends: kv_default: type: kv_sqlite db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/open-benchmark}/kvstore.db sql_default: type: sql_sqlite db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/open-benchmark}/sql_store.db stores: metadata: namespace: registry backend: kv_default inference: table_name: inference_store backend: sql_default max_write_queue_size: 10000 num_writers: 4 conversations: table_name: openai_conversations backend: sql_default registered_resources: models: - metadata: {} model_id: gpt-4o provider_id: openai provider_model_id: gpt-4o model_type: llm - metadata: {} model_id: claude-3-5-sonnet-latest provider_id: anthropic provider_model_id: claude-3-5-sonnet-latest model_type: llm - metadata: {} model_id: gemini/gemini-1.5-flash provider_id: gemini provider_model_id: gemini/gemini-1.5-flash model_type: llm - metadata: {} model_id: meta-llama/Llama-3.3-70B-Instruct provider_id: groq provider_model_id: groq/llama-3.3-70b-versatile model_type: llm - metadata: {} model_id: meta-llama/Llama-3.1-405B-Instruct provider_id: together provider_model_id: meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo model_type: llm shields: - shield_id: meta-llama/Llama-Guard-3-8B vector_dbs: [] datasets: - purpose: eval/messages-answer source: type: uri uri: huggingface://datasets/llamastack/simpleqa?split=train metadata: {} dataset_id: simpleqa - purpose: eval/messages-answer source: type: uri uri: huggingface://datasets/llamastack/mmlu_cot?split=test&name=all metadata: {} dataset_id: mmlu_cot - purpose: eval/messages-answer source: type: uri uri: huggingface://datasets/llamastack/gpqa_0shot_cot?split=test&name=gpqa_main metadata: {} dataset_id: gpqa_cot - purpose: eval/messages-answer source: type: uri uri: huggingface://datasets/llamastack/math_500?split=test metadata: {} dataset_id: math_500 - purpose: eval/messages-answer source: type: uri uri: huggingface://datasets/llamastack/IfEval?split=train metadata: {} dataset_id: ifeval - purpose: eval/messages-answer source: type: uri uri: huggingface://datasets/llamastack/docvqa?split=val metadata: {} dataset_id: docvqa scoring_fns: [] benchmarks: - dataset_id: simpleqa scoring_functions: - llm-as-judge::405b-simpleqa metadata: {} benchmark_id: meta-reference-simpleqa - dataset_id: mmlu_cot scoring_functions: - basic::regex_parser_multiple_choice_answer metadata: {} benchmark_id: meta-reference-mmlu-cot - dataset_id: gpqa_cot scoring_functions: - basic::regex_parser_multiple_choice_answer metadata: {} benchmark_id: meta-reference-gpqa-cot - dataset_id: math_500 scoring_functions: - basic::regex_parser_math_response metadata: {} benchmark_id: meta-reference-math-500 - dataset_id: ifeval scoring_functions: - basic::ifeval metadata: {} benchmark_id: meta-reference-ifeval - dataset_id: docvqa scoring_functions: - basic::docvqa metadata: {} benchmark_id: meta-reference-docvqa tool_groups: - toolgroup_id: builtin::websearch provider_id: tavily-search server: port: 8321 telemetry: enabled: true