diff --git a/.github/workflows/publish-to-docker.yml b/.github/workflows/publish-to-docker.yml new file mode 100644 index 000000000..1010041b7 --- /dev/null +++ b/.github/workflows/publish-to-docker.yml @@ -0,0 +1,138 @@ +name: Docker Build and Publish + +on: + workflow_dispatch: + inputs: + version: + description: 'TestPyPI or PyPI version to build (e.g., 0.0.63.dev20250114)' + required: true + type: string + +jobs: + build-and-push: + runs-on: ubuntu-latest + env: + TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} + FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }} + TAVILY_SEARCH_API_KEY: ${{ secrets.TAVILY_SEARCH_API_KEY }} + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set version + id: version + run: | + if [ "${{ github.event_name }}" = "push" ]; then + echo "VERSION=0.0.63.dev51206766" >> $GITHUB_OUTPUT + else + echo "VERSION=${{ inputs.version }}" >> $GITHUB_OUTPUT + fi + + - name: Check package version availability + run: | + # Function to check if version exists in a repository + check_version() { + local repo=$1 + local status_code=$(curl -s -o /dev/null -w "%{http_code}" "https://$repo.org/project/llama-stack/${{ steps.version.outputs.version }}") + return $([ "$status_code" -eq 200 ]) + } + + # Check TestPyPI first, then PyPI + if check_version "test.pypi"; then + echo "Version ${{ steps.version.outputs.version }} found in TestPyPI" + echo "PYPI_SOURCE=testpypi" >> $GITHUB_ENV + elif check_version "pypi"; then + echo "Version ${{ steps.version.outputs.version }} found in PyPI" + echo "PYPI_SOURCE=pypi" >> $GITHUB_ENV + else + echo "Error: Version ${{ steps.version.outputs.version }} not found in either TestPyPI or PyPI" + exit 1 + fi + + - name: Install llama-stack + run: | + if [ "${{ github.event_name }}" = "push" ]; then + pip install -e . + else + if [ "$PYPI_SOURCE" = "testpypi" ]; then + pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple llama-stack==${{ steps.version.outputs.version }} + else + pip install llama-stack==${{ steps.version.outputs.version }} + fi + fi + + - name: Build docker image + run: | + TEMPLATES=("ollama" "bedrock" "remote-vllm" "fireworks" "together" "tgi" "meta-reference-gpu") + for template in "${TEMPLATES[@]}"; do + if [ "$PYPI_SOURCE" = "testpypi" ]; then + TEST_PYPI_VERSION=${{ steps.version.outputs.version }} llama stack build --template $template --image-type container + else + PYPI_VERSION=${{ steps.version.outputs.version }} llama stack build --template $template --image-type container + fi + done + + - name: List docker images + run: | + docker images + + # TODO (xiyan): make the following 2 steps into a matrix and test all templates other than fireworks + - name: Start up built docker image + run: | + cd distributions/fireworks + if [ "$PYPI_SOURCE" = "testpypi" ]; then + sed -i 's|image: llamastack/distribution-fireworks|image: distribution-fireworks:test-${{ steps.version.outputs.version }}|' ./compose.yaml + else + sed -i 's|image: llamastack/distribution-fireworks|image: distribution-fireworks:${{ steps.version.outputs.version }}|' ./compose.yaml + fi + docker compose up -d + cd .. + # Wait for the container to start + timeout=300 + while ! curl -s -f http://localhost:8321/v1/version > /dev/null && [ $timeout -gt 0 ]; do + echo "Waiting for endpoint to be available..." + sleep 5 + timeout=$((timeout - 5)) + done + + if [ $timeout -le 0 ]; then + echo "Timeout waiting for endpoint to become available" + exit 1 + fi + + - name: Run simple models list test on docker server + run: | + curl http://localhost:8321/v1/models + + # TODO (xiyan): figure out why client cannot find server but curl works + # - name: Run pytest on docker server + # run: | + # pip install pytest pytest-md-report + # export LLAMA_STACK_BASE_URL="http://localhost:8321" + # LLAMA_STACK_BASE_URL="http://localhost:8321" pytest -v tests/client-sdk/inference/test_inference.py --md-report --md-report-verbose=1 + + - name: Push to dockerhub + run: | + TEMPLATES=("ollama" "bedrock" "remote-vllm" "fireworks" "together" "tgi" "meta-reference-gpu") + for template in "${TEMPLATES[@]}"; do + if [ "$PYPI_SOURCE" = "testpypi" ]; then + docker tag distribution-$template:test-${{ steps.version.outputs.version }} llamastack/distribution-$template:test-${{ steps.version.outputs.version }} + docker push llamastack/distribution-$template:test-${{ steps.version.outputs.version }} + else + docker tag distribution-$template:${{ steps.version.outputs.version }} llamastack/distribution-$template:${{ steps.version.outputs.version }} + docker push llamastack/distribution-$template:${{ steps.version.outputs.version }} + fi + done diff --git a/.github/workflows/publish-to-test-pypi.yml b/.github/workflows/publish-to-test-pypi.yml index 957761235..2e8aaab23 100644 --- a/.github/workflows/publish-to-test-pypi.yml +++ b/.github/workflows/publish-to-test-pypi.yml @@ -199,7 +199,13 @@ jobs: - publish-to-testpypi - trigger-client-and-models-build runs-on: ubuntu-latest + env: + TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} + TAVILY_SEARCH_API_KEY: ${{ secrets.TAVILY_SEARCH_API_KEY }} steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Install the package run: | max_attempts=6 @@ -228,5 +234,11 @@ jobs: llama stack list-apis llama stack list-providers inference llama stack list-providers telemetry + - name: Test Notebook + run: | + pip install pytest nbval + llama stack build --template together --image-type venv + pytest -v -s --nbval-lax ./docs/getting_started.ipynb + pytest -v -s --nbval-lax ./docs/notebooks/Llama_Stack_Benchmark_Evals.ipynb # TODO: add trigger for integration test workflow & docker builds diff --git a/README.md b/README.md index 61a0f33fe..17acd0096 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,15 @@ [![PyPI - Downloads](https://img.shields.io/pypi/dm/llama-stack)](https://pypi.org/project/llama-stack/) [![Discord](https://img.shields.io/discord/1257833999603335178)](https://discord.gg/llama-stack) -[**Quick Start**](https://llama-stack.readthedocs.io/en/latest/getting_started/index.html) | [**Documentation**](https://llama-stack.readthedocs.io/en/latest/index.html) | [**Zero-to-Hero Guide**](https://github.com/meta-llama/llama-stack/tree/main/docs/zero_to_hero_guide) +[**Quick Start**](https://llama-stack.readthedocs.io/en/latest/getting_started/index.html) | [**Documentation**](https://llama-stack.readthedocs.io/en/latest/index.html) | [**Colab Notebook**](./docs/getting_started.ipynb) -Llama Stack defines and standardizes the set of core building blocks needed to bring generative AI applications to market. These building blocks are presented in the form of interoperable APIs with a broad set of Service Providers providing their implementations. +Llama Stack defines and standardizes the core building blocks that simplify AI application development. It codified best practices across the Llama ecosystem. More specifically, it provides + +- **Unified API layer** for Inference, RAG, Agents, Tools, Safety, Evals, and Telemetry. +- **Plugin architecture** to support the rich ecosystem of implementations of the different APIs in different environments like local development, on-premises, cloud, and mobile. +- **Prepackaged verified distributions** which offer a one-stop solution for developers to get started quickly and reliably in any environment +- **Multiple developer interfaces** like CLI and SDKs for Python, Node, iOS, and Android +- **Standalone applications** as examples for how to build production-grade AI applications with Llama Stack
-Our goal is to provide pre-packaged implementations which can be operated in a variety of deployment environments: developers start iterating with Desktops or their mobile devices and can seamlessly transition to on-prem or public cloud deployments. At every point in this transition, the same set of APIs and the same developer experience is available. +### Llama Stack Benefits +- **Flexible Options**: Developers can choose their preferred infrastructure without changing APIs and enjoy flexible deployment choice. +- **Consistent Experience**: With its unified APIs Llama Stack makes it easier to build, test, and deploy AI applications with consistent application behavior. +- **Robust Ecosystem**: Llama Stack is already integrated with distribution partners (cloud providers, hardware vendors, and AI-focused companies) that offer tailored infrastructure, software, and services for deploying Llama models. -> ⚠️ **Note** -> The Stack APIs are rapidly improving, but still very much work in progress and we invite feedback as well as direct contributions. +By reducing friction and complexity, Llama Stack empowers developers to focus on what they do best: building transformative generative AI applications. - -## APIs - -We have working implementations of the following APIs today: -- Inference -- Safety -- Memory -- Agents -- Eval -- Telemetry - -Alongside these APIs, we also related APIs for operating with associated resources (see [Concepts](https://llama-stack.readthedocs.io/en/latest/concepts/index.html#resources)): - -- Models -- Shields -- Memory Banks -- Eval Tasks -- Datasets -- Scoring Functions - -We are also working on the following APIs which will be released soon: - -- Post Training -- Synthetic Data Generation -- Reward Scoring - -Each of the APIs themselves is a collection of REST endpoints. - -## Philosophy - -### Service-oriented design - -Unlike other frameworks, Llama Stack is built with a service-oriented, REST API-first approach. Such a design not only allows for seamless transitions from a local to remote deployments, but also forces the design to be more declarative. We believe this restriction can result in a much simpler, robust developer experience. This will necessarily trade-off against expressivity however if we get the APIs right, it can lead to a very powerful platform. - -### Composability - -We expect the set of APIs we design to be composable. An Agent abstractly depends on { Inference, Memory, Safety } APIs but does not care about the actual implementation details. Safety itself may require model inference and hence can depend on the Inference API. - -### Turnkey one-stop solutions - -We expect to provide turnkey solutions for popular deployment scenarios. It should be easy to deploy a Llama Stack server on AWS or on a private data center. Either of these should allow a developer to get started with powerful agentic apps, model evaluations or fine-tuning services in a matter of minutes. They should all result in the same uniform observability and developer experience. - -### Focus on Llama models - -As a Meta initiated project, we have started by explicitly focusing on Meta's Llama series of models. Supporting the broad set of open models is no easy task and we want to start with models we understand best. - -### Supporting the Ecosystem - -There is a vibrant ecosystem of Providers which provide efficient inference or scalable vector stores or powerful observability solutions. We want to make sure it is easy for developers to pick and choose the best implementations for their use cases. We also want to make sure it is easy for new Providers to onboard and participate in the ecosystem. - -Additionally, we have designed every element of the Stack such that APIs as well as Resources (like Models) can be federated. - - -## Supported Llama Stack Implementations ### API Providers +Here is a list of the various API providers and available distributions to developers started easily, + | **API Provider Builder** | **Environments** | **Agents** | **Inference** | **Memory** | **Safety** | **Telemetry** | |:------------------------------------------------------------------------------------------:|:----------------------:|:------------------:|:------------------:|:------------------:|:------------------:|:------------------:| | Meta Reference | Single Node | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | @@ -87,14 +43,16 @@ Additionally, we have designed every element of the Stack such that APIs as well | Groq | Hosted | | :heavy_check_mark: | | | | | Ollama | Single Node | | :heavy_check_mark: | | | | | TGI | Hosted and Single Node | | :heavy_check_mark: | | | | -| [NVIDIA NIM](https://build.nvidia.com/nim?filters=nimType%3Anim_type_run_anywhere&q=llama) | Hosted and Single Node | | :heavy_check_mark: | | | | +| NVIDIA NIM | Hosted and Single Node | | :heavy_check_mark: | | | | | Chroma | Single Node | | | :heavy_check_mark: | | | | PG Vector | Single Node | | | :heavy_check_mark: | | | | PyTorch ExecuTorch | On-device iOS | :heavy_check_mark: | :heavy_check_mark: | | | | -| [vLLM](https://github.com/vllm-project/vllm) | Hosted and Single Node | | :heavy_check_mark: | | | | +| vLLM | Hosted and Single Node | | :heavy_check_mark: | | | | ### Distributions +A Llama Stack Distribution (or "distro") is a pre-configured bundle of provider implementations for each API component. Distributions make it easy to get started with a specific deployment scenario - you can begin with a local development setup (eg. ollama) and seamlessly transition to production (eg. Fireworks) without changing your application code. Here are some of the distributions we support: + | **Distribution** | **Llama Stack Docker** | Start This Distribution | |:---------------------------------------------:|:-------------------------------------------------------------------------------------------------------------------------------------------------------------:|:------------------------------------------------------------------------------------------------------------------------:| | Meta Reference | [llamastack/distribution-meta-reference-gpu](https://hub.docker.com/repository/docker/llamastack/distribution-meta-reference-gpu/general) | [Guide](https://llama-stack.readthedocs.io/en/latest/distributions/self_hosted_distro/meta-reference-gpu.html) | @@ -104,9 +62,9 @@ Additionally, we have designed every element of the Stack such that APIs as well | TGI | [llamastack/distribution-tgi](https://hub.docker.com/repository/docker/llamastack/distribution-tgi/general) | [Guide](https://llama-stack.readthedocs.io/en/latest/distributions/self_hosted_distro/tgi.html) | | Together | [llamastack/distribution-together](https://hub.docker.com/repository/docker/llamastack/distribution-together/general) | [Guide](https://llama-stack.readthedocs.io/en/latest/distributions/self_hosted_distro/together.html) | | Fireworks | [llamastack/distribution-fireworks](https://hub.docker.com/repository/docker/llamastack/distribution-fireworks/general) | [Guide](https://llama-stack.readthedocs.io/en/latest/distributions/self_hosted_distro/fireworks.html) | -| [vLLM](https://github.com/vllm-project/vllm) | [llamastack/distribution-remote-vllm](https://hub.docker.com/repository/docker/llamastack/distribution-remote-vllm/general) | [Guide](https://llama-stack.readthedocs.io/en/latest/distributions/self_hosted_distro/remote-vllm.html) | +| vLLM | [llamastack/distribution-remote-vllm](https://hub.docker.com/repository/docker/llamastack/distribution-remote-vllm/general) | [Guide](https://llama-stack.readthedocs.io/en/latest/distributions/self_hosted_distro/remote-vllm.html) | -## Installation +### Installation You have two ways to install this repository: @@ -131,7 +89,7 @@ You have two ways to install this repository: pip install -e . ``` -## Documentation +### Documentation Please checkout our [Documentation](https://llama-stack.readthedocs.io/en/latest/index.html) page for more details. @@ -139,13 +97,13 @@ Please checkout our [Documentation](https://llama-stack.readthedocs.io/en/latest * Guide using `llama` CLI to work with Llama models (download, study prompts), and building/starting a Llama Stack distribution. * [Getting Started](https://llama-stack.readthedocs.io/en/latest/getting_started/index.html) * Quick guide to start a Llama Stack server. - * [Jupyter notebook](./docs/notebooks/Llama_Stack_Building_AI_Applications.ipynb) to walk-through how to use simple text and vision inference llama_stack_client APIs + * [Jupyter notebook](./docs/getting_started.ipynb) to walk-through how to use simple text and vision inference llama_stack_client APIs * The complete Llama Stack lesson [Colab notebook](https://colab.research.google.com/drive/1dtVmxotBsI4cGZQNsJRYPrLiDeT0Wnwt) of the new [Llama 3.2 course on Deeplearning.ai](https://learn.deeplearning.ai/courses/introducing-multimodal-llama-3-2/lesson/8/llama-stack). * A [Zero-to-Hero Guide](https://github.com/meta-llama/llama-stack/tree/main/docs/zero_to_hero_guide) that guide you through all the key components of llama stack with code samples. * [Contributing](CONTRIBUTING.md) * [Adding a new API Provider](https://llama-stack.readthedocs.io/en/latest/contributing/new_api_provider.html) to walk-through how to add a new API provider. -## Llama Stack Client SDKs +### Llama Stack Client SDKs | **Language** | **Client SDK** | **Package** | | :----: | :----: | :----: | diff --git a/distributions/bedrock/compose.yaml b/distributions/bedrock/compose.yaml index f988e33d1..055b92c67 100644 --- a/distributions/bedrock/compose.yaml +++ b/distributions/bedrock/compose.yaml @@ -5,7 +5,7 @@ services: - ~/.llama:/root/.llama - ./run.yaml:/root/llamastack-run-bedrock.yaml ports: - - "5000:5000" + - "8321:8321" entrypoint: bash -c "python -m llama_stack.distribution.server.server --yaml_config /root/llamastack-run-bedrock.yaml" deploy: restart_policy: diff --git a/distributions/cerebras/compose.yaml b/distributions/cerebras/compose.yaml index f2e9a6f42..8dc09a865 100644 --- a/distributions/cerebras/compose.yaml +++ b/distributions/cerebras/compose.yaml @@ -6,7 +6,7 @@ services: - ~/.llama:/root/.llama - ./run.yaml:/root/llamastack-run-cerebras.yaml ports: - - "5000:5000" + - "8321:8321" entrypoint: bash -c "python -m llama_stack.distribution.server.server --yaml_config /root/llamastack-run-cerebras.yaml" deploy: restart_policy: diff --git a/distributions/dell-tgi/compose.yaml b/distributions/dell-tgi/compose.yaml index 0e325aff5..d26636cbd 100644 --- a/distributions/dell-tgi/compose.yaml +++ b/distributions/dell-tgi/compose.yaml @@ -40,7 +40,7 @@ services: # Link to TGI run.yaml file - ./run.yaml:/root/my-run.yaml ports: - - "5000:5000" + - "8321:8321" # Hack: wait for TGI server to start before starting docker entrypoint: bash -c "sleep 60; python -m llama_stack.distribution.server.server --yaml_config /root/my-run.yaml" restart_policy: diff --git a/distributions/dell-tgi/run.yaml b/distributions/dell-tgi/run.yaml index 3f8a98779..cd6ddcfdf 100644 --- a/distributions/dell-tgi/run.yaml +++ b/distributions/dell-tgi/run.yaml @@ -1,6 +1,6 @@ version: '2' image_name: local -docker_image: null +container_image: null conda_env: local apis: - shields diff --git a/distributions/dependencies.json b/distributions/dependencies.json index bd363ea40..7b5d8b002 100644 --- a/distributions/dependencies.json +++ b/distributions/dependencies.json @@ -13,6 +13,7 @@ "httpx", "huggingface_hub", "matplotlib", + "mcp", "nltk", "numpy", "openai", @@ -45,6 +46,7 @@ "fire", "httpx", "matplotlib", + "mcp", "nltk", "numpy", "openai", @@ -78,6 +80,7 @@ "fire", "httpx", "matplotlib", + "mcp", "nltk", "numpy", "openai", @@ -101,14 +104,17 @@ ], "remote-vllm": [ "aiosqlite", + "autoevals", "blobfile", "chardet", "chromadb-client", + "datasets", "faiss-cpu", "fastapi", "fire", "httpx", "matplotlib", + "mcp", "nltk", "numpy", "openai", @@ -142,6 +148,7 @@ "fireworks-ai", "httpx", "matplotlib", + "mcp", "nltk", "numpy", "openai", @@ -176,6 +183,7 @@ "httpx", "huggingface_hub", "matplotlib", + "mcp", "nltk", "numpy", "openai", @@ -209,6 +217,7 @@ "fire", "httpx", "matplotlib", + "mcp", "nltk", "numpy", "openai", @@ -244,6 +253,7 @@ "httpx", "lm-format-enforcer", "matplotlib", + "mcp", "nltk", "numpy", "openai", @@ -268,6 +278,38 @@ "sentence-transformers --no-deps", "torch --index-url https://download.pytorch.org/whl/cpu" ], + "nvidia": [ + "aiosqlite", + "autoevals", + "blobfile", + "chardet", + "datasets", + "faiss-cpu", + "fastapi", + "fire", + "httpx", + "matplotlib", + "mcp", + "nltk", + "numpy", + "openai", + "opentelemetry-exporter-otlp-proto-http", + "opentelemetry-sdk", + "pandas", + "pillow", + "psycopg2-binary", + "pypdf", + "redis", + "requests", + "scikit-learn", + "scipy", + "sentencepiece", + "tqdm", + "transformers", + "uvicorn", + "sentence-transformers --no-deps", + "torch --index-url https://download.pytorch.org/whl/cpu" + ], "meta-reference-quantized-gpu": [ "accelerate", "aiosqlite", @@ -284,6 +326,7 @@ "httpx", "lm-format-enforcer", "matplotlib", + "mcp", "nltk", "numpy", "openai", @@ -311,9 +354,12 @@ ], "cerebras": [ "aiosqlite", + "autoevals", "blobfile", "cerebras_cloud_sdk", "chardet", + "chromadb-client", + "datasets", "faiss-cpu", "fastapi", "fire", @@ -321,6 +367,7 @@ "matplotlib", "nltk", "numpy", + "openai", "opentelemetry-exporter-otlp-proto-http", "opentelemetry-sdk", "pandas", @@ -386,6 +433,7 @@ "httpx", "huggingface_hub", "matplotlib", + "mcp", "nltk", "numpy", "openai", diff --git a/distributions/fireworks/compose.yaml b/distributions/fireworks/compose.yaml index 71137c040..84b8491e4 100644 --- a/distributions/fireworks/compose.yaml +++ b/distributions/fireworks/compose.yaml @@ -1,13 +1,11 @@ services: llamastack: image: llamastack/distribution-fireworks - network_mode: "host" - volumes: - - ~/.llama:/root/.llama - - ./run.yaml:/root/llamastack-run-fireworks.yaml ports: - - "5000:5000" - entrypoint: bash -c "python -m llama_stack.distribution.server.server --yaml_config /root/llamastack-run-fireworks.yaml" + - "8321:8321" + environment: + - FIREWORKS_API_KEY=${FIREWORKS_API_KEY} + entrypoint: bash -c "python -m llama_stack.distribution.server.server --template fireworks" deploy: restart_policy: condition: on-failure diff --git a/distributions/meta-reference-gpu/compose.yaml b/distributions/meta-reference-gpu/compose.yaml index 2b88c68fc..d977e92ea 100644 --- a/distributions/meta-reference-gpu/compose.yaml +++ b/distributions/meta-reference-gpu/compose.yaml @@ -6,7 +6,7 @@ services: - ~/.llama:/root/.llama - ./run.yaml:/root/my-run.yaml ports: - - "5000:5000" + - "8321:8321" devices: - nvidia.com/gpu=all environment: diff --git a/distributions/meta-reference-quantized-gpu/compose.yaml b/distributions/meta-reference-quantized-gpu/compose.yaml index f9fe9f45d..98e943dce 100644 --- a/distributions/meta-reference-quantized-gpu/compose.yaml +++ b/distributions/meta-reference-quantized-gpu/compose.yaml @@ -6,7 +6,7 @@ services: - ~/.llama:/root/.llama - ./run.yaml:/root/my-run.yaml ports: - - "5000:5000" + - "8321:8321" devices: - nvidia.com/gpu=all environment: diff --git a/distributions/meta-reference-quantized-gpu/run.yaml b/distributions/meta-reference-quantized-gpu/run.yaml index 19c726b09..eb631adaa 100644 --- a/distributions/meta-reference-quantized-gpu/run.yaml +++ b/distributions/meta-reference-quantized-gpu/run.yaml @@ -1,6 +1,6 @@ version: '2' image_name: local -docker_image: null +container_image: null conda_env: local apis: - shields diff --git a/distributions/remote-nvidia/build.yaml b/distributions/remote-nvidia/build.yaml new file mode 120000 index 000000000..8903d2e57 --- /dev/null +++ b/distributions/remote-nvidia/build.yaml @@ -0,0 +1 @@ +../../llama_stack/templates/nvidia/build.yaml \ No newline at end of file diff --git a/distributions/remote-nvidia/compose.yaml b/distributions/remote-nvidia/compose.yaml new file mode 100644 index 000000000..ab8b4ce25 --- /dev/null +++ b/distributions/remote-nvidia/compose.yaml @@ -0,0 +1,19 @@ +services: + llamastack: + image: distribution-nvidia:dev + network_mode: "host" + volumes: + - ~/.llama:/root/.llama + - ./run.yaml:/root/llamastack-run-nvidia.yaml + ports: + - "8321:8321" + environment: + - INFERENCE_MODEL=${INFERENCE_MODEL:-Llama3.1-8B-Instruct} + - NVIDIA_API_KEY=${NVIDIA_API_KEY:-} + entrypoint: bash -c "python -m llama_stack.distribution.server.server --yaml-config /root/llamastack-run-nvidia.yaml" + deploy: + restart_policy: + condition: on-failure + delay: 3s + max_attempts: 5 + window: 60s diff --git a/distributions/remote-nvidia/run.yaml b/distributions/remote-nvidia/run.yaml new file mode 120000 index 000000000..85da3e26b --- /dev/null +++ b/distributions/remote-nvidia/run.yaml @@ -0,0 +1 @@ +../../llama_stack/templates/nvidia/run.yaml \ No newline at end of file diff --git a/distributions/together/compose.yaml b/distributions/together/compose.yaml index 8d938990e..f66ee69f9 100644 --- a/distributions/together/compose.yaml +++ b/distributions/together/compose.yaml @@ -1,13 +1,11 @@ services: llamastack: image: llamastack/distribution-together - network_mode: "host" - volumes: - - ~/.llama:/root/.llama - - ./run.yaml:/root/llamastack-run-together.yaml ports: - - "5000:5000" - entrypoint: bash -c "python -m llama_stack.distribution.server.server --yaml_config /root/llamastack-run-together.yaml" + - "8321:8321" + environment: + - TOGETHER_API_KEY=${TOGETHER_API_KEY} + entrypoint: bash -c "python -m llama_stack.distribution.server.server --template together" deploy: restart_policy: condition: on-failure diff --git a/distributions/vllm-gpu/compose.yaml b/distributions/vllm-gpu/compose.yaml index f8779c9ce..98267cdc3 100644 --- a/distributions/vllm-gpu/compose.yaml +++ b/distributions/vllm-gpu/compose.yaml @@ -6,7 +6,7 @@ services: - ~/.llama:/root/.llama - ./run.yaml:/root/my-run.yaml ports: - - "5000:5000" + - "8321:8321" devices: - nvidia.com/gpu=all environment: diff --git a/distributions/vllm-gpu/run.yaml b/distributions/vllm-gpu/run.yaml index f42c942a3..a75a4c451 100644 --- a/distributions/vllm-gpu/run.yaml +++ b/distributions/vllm-gpu/run.yaml @@ -1,6 +1,6 @@ version: '2' image_name: local -docker_image: null +container_image: null conda_env: local apis: - shields diff --git a/docs/getting_started.ipynb b/docs/getting_started.ipynb index fa527f1a0..0c0f7fa95 100644 --- a/docs/getting_started.ipynb +++ b/docs/getting_started.ipynb @@ -71,7 +71,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 1, "id": "J2kGed0R5PSf", "metadata": { "colab": { @@ -79,7 +79,7 @@ }, "collapsed": true, "id": "J2kGed0R5PSf", - "outputId": "7d543c6f-623d-4911-b9a7-4ed24d5b82f2" + "outputId": "2478ea60-8d35-48a1-b011-f233831740c5" }, "outputs": [ { @@ -89,65 +89,111 @@ "Reading package lists... Done\n", "Building dependency tree... Done\n", "Reading state information... Done\n", - "bubblewrap is already the newest version (0.6.1-1ubuntu0.1).\n", - "0 upgraded, 0 newly installed, 0 to remove and 49 not upgraded.\n", - "Requirement already satisfied: llama-stack in /usr/local/lib/python3.10/dist-packages (0.0.61)\n", - "Requirement already satisfied: blobfile in /usr/local/lib/python3.10/dist-packages (from llama-stack) (3.0.0)\n", - "Requirement already satisfied: fire in /usr/local/lib/python3.10/dist-packages (from llama-stack) (0.7.0)\n", - "Requirement already satisfied: httpx in /usr/local/lib/python3.10/dist-packages (from llama-stack) (0.28.1)\n", - "Requirement already satisfied: huggingface-hub in /usr/local/lib/python3.10/dist-packages (from llama-stack) (0.26.5)\n", - "Requirement already satisfied: llama-models>=0.0.61 in /usr/local/lib/python3.10/dist-packages (from llama-stack) (0.0.61)\n", - "Requirement already satisfied: llama-stack-client>=0.0.61 in /usr/local/lib/python3.10/dist-packages (from llama-stack) (0.0.61)\n", - "Requirement already satisfied: prompt-toolkit in /usr/local/lib/python3.10/dist-packages (from llama-stack) (3.0.48)\n", - "Requirement already satisfied: python-dotenv in /usr/local/lib/python3.10/dist-packages (from llama-stack) (1.0.1)\n", - "Requirement already satisfied: pydantic>=2 in /usr/local/lib/python3.10/dist-packages (from llama-stack) (2.10.3)\n", - "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from llama-stack) (2.32.3)\n", - "Requirement already satisfied: rich in /usr/local/lib/python3.10/dist-packages (from llama-stack) (13.9.4)\n", - "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from llama-stack) (75.1.0)\n", - "Requirement already satisfied: termcolor in /usr/local/lib/python3.10/dist-packages (from llama-stack) (2.5.0)\n", - "Requirement already satisfied: PyYAML in /usr/local/lib/python3.10/dist-packages (from llama-models>=0.0.61->llama-stack) (6.0.2)\n", - "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from llama-models>=0.0.61->llama-stack) (3.1.4)\n", - "Requirement already satisfied: tiktoken in /usr/local/lib/python3.10/dist-packages (from llama-models>=0.0.61->llama-stack) (0.8.0)\n", - "Requirement already satisfied: Pillow in /usr/local/lib/python3.10/dist-packages (from llama-models>=0.0.61->llama-stack) (10.4.0)\n", - "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.61->llama-stack) (3.7.1)\n", - "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.61->llama-stack) (8.1.7)\n", - "Requirement already satisfied: distro<2,>=1.7.0 in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.61->llama-stack) (1.9.0)\n", - "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.61->llama-stack) (2.2.2)\n", - "Requirement already satisfied: pyaml in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.61->llama-stack) (24.12.1)\n", - "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.61->llama-stack) (1.3.1)\n", - "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.61->llama-stack) (4.66.6)\n", - "Requirement already satisfied: typing-extensions<5,>=4.7 in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.61->llama-stack) (4.12.2)\n", - "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx->llama-stack) (2024.8.30)\n", - "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx->llama-stack) (1.0.7)\n", - "Requirement already satisfied: idna in /usr/local/lib/python3.10/dist-packages (from httpx->llama-stack) (3.10)\n", - "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx->llama-stack) (0.14.0)\n", - "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.10/dist-packages (from pydantic>=2->llama-stack) (0.7.0)\n", - "Requirement already satisfied: pydantic-core==2.27.1 in /usr/local/lib/python3.10/dist-packages (from pydantic>=2->llama-stack) (2.27.1)\n", - "Requirement already satisfied: pycryptodomex>=3.8 in /usr/local/lib/python3.10/dist-packages (from blobfile->llama-stack) (3.21.0)\n", - "Requirement already satisfied: urllib3<3,>=1.25.3 in /usr/local/lib/python3.10/dist-packages (from blobfile->llama-stack) (2.2.3)\n", - "Requirement already satisfied: lxml>=4.9 in /usr/local/lib/python3.10/dist-packages (from blobfile->llama-stack) (5.3.0)\n", - "Requirement already satisfied: filelock>=3.0 in /usr/local/lib/python3.10/dist-packages (from blobfile->llama-stack) (3.16.1)\n", - "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->llama-stack) (2024.9.0)\n", - "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->llama-stack) (24.2)\n", - "Requirement already satisfied: wcwidth in /usr/local/lib/python3.10/dist-packages (from prompt-toolkit->llama-stack) (0.2.13)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->llama-stack) (3.4.0)\n", - "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich->llama-stack) (3.0.0)\n", - "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich->llama-stack) (2.18.0)\n", - "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->llama-stack-client>=0.0.61->llama-stack) (1.2.2)\n", - "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich->llama-stack) (0.1.2)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->llama-models>=0.0.61->llama-stack) (3.0.2)\n", - "Requirement already satisfied: numpy>=1.22.4 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-stack-client>=0.0.61->llama-stack) (1.26.4)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-stack-client>=0.0.61->llama-stack) (2.8.2)\n", - "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-stack-client>=0.0.61->llama-stack) (2024.2)\n", - "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-stack-client>=0.0.61->llama-stack) (2024.2)\n", - "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken->llama-models>=0.0.61->llama-stack) (2024.9.11)\n", - "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas->llama-stack-client>=0.0.61->llama-stack) (1.17.0)\n" + "The following NEW packages will be installed:\n", + " bubblewrap\n", + "0 upgraded, 1 newly installed, 0 to remove and 49 not upgraded.\n", + "Need to get 46.3 kB of archives.\n", + "After this operation, 132 kB of additional disk space will be used.\n", + "Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 bubblewrap amd64 0.6.1-1ubuntu0.1 [46.3 kB]\n", + "Fetched 46.3 kB in 0s (122 kB/s)\n", + "Selecting previously unselected package bubblewrap.\n", + "(Reading database ... 124561 files and directories currently installed.)\n", + "Preparing to unpack .../bubblewrap_0.6.1-1ubuntu0.1_amd64.deb ...\n", + "Unpacking bubblewrap (0.6.1-1ubuntu0.1) ...\n", + "Setting up bubblewrap (0.6.1-1ubuntu0.1) ...\n", + "Processing triggers for man-db (2.10.2-1) ...\n", + "Looking in indexes: https://test.pypi.org/simple/, https://pypi.python.org/simple\n", + "Collecting llama-stack==0.1.0rc10\n", + " Downloading https://test-files.pythonhosted.org/packages/68/22/4a170fbe01095df81e76c7bf8f35c716c1a0a5ec4503da6e78695fab351c/llama_stack-0.1.0rc10-py3-none-any.whl.metadata (15 kB)\n", + "Collecting blobfile (from llama-stack==0.1.0rc10)\n", + " Downloading blobfile-3.0.0-py3-none-any.whl.metadata (15 kB)\n", + "Collecting fire (from llama-stack==0.1.0rc10)\n", + " Downloading fire-0.7.0.tar.gz (87 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m87.2/87.2 kB\u001b[0m \u001b[31m4.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + "Requirement already satisfied: httpx in /usr/local/lib/python3.11/dist-packages (from llama-stack==0.1.0rc10) (0.28.1)\n", + "Requirement already satisfied: huggingface-hub in /usr/local/lib/python3.11/dist-packages (from llama-stack==0.1.0rc10) (0.27.1)\n", + "Collecting llama-models==0.1.0rc10 (from llama-stack==0.1.0rc10)\n", + " Downloading https://test-files.pythonhosted.org/packages/45/2b/6a6947d5915054b9980f82606942f1b79960a27168299254ca12e5b5795b/llama_models-0.1.0rc10-py3-none-any.whl.metadata (8.5 kB)\n", + "Collecting llama-stack-client==0.1.0rc10 (from llama-stack==0.1.0rc10)\n", + " Downloading https://test-files.pythonhosted.org/packages/d6/85/a4fd621c4ae4db7339ab098b37bf4b4ad3cc12440e75ef10ec524e28ef7d/llama_stack_client-0.1.0rc10-py3-none-any.whl.metadata (15 kB)\n", + "Requirement already satisfied: prompt-toolkit in /usr/local/lib/python3.11/dist-packages (from llama-stack==0.1.0rc10) (3.0.48)\n", + "Collecting python-dotenv (from llama-stack==0.1.0rc10)\n", + " Downloading python_dotenv-1.0.1-py3-none-any.whl.metadata (23 kB)\n", + "Requirement already satisfied: pydantic>=2 in /usr/local/lib/python3.11/dist-packages (from llama-stack==0.1.0rc10) (2.10.5)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.11/dist-packages (from llama-stack==0.1.0rc10) (2.32.3)\n", + "Requirement already satisfied: rich in /usr/local/lib/python3.11/dist-packages (from llama-stack==0.1.0rc10) (13.9.4)\n", + "Requirement already satisfied: setuptools in /usr/local/lib/python3.11/dist-packages (from llama-stack==0.1.0rc10) (75.1.0)\n", + "Requirement already satisfied: termcolor in /usr/local/lib/python3.11/dist-packages (from llama-stack==0.1.0rc10) (2.5.0)\n", + "Requirement already satisfied: PyYAML in /usr/local/lib/python3.11/dist-packages (from llama-models==0.1.0rc10->llama-stack==0.1.0rc10) (6.0.2)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.11/dist-packages (from llama-models==0.1.0rc10->llama-stack==0.1.0rc10) (3.1.5)\n", + "Collecting tiktoken (from llama-models==0.1.0rc10->llama-stack==0.1.0rc10)\n", + " Downloading tiktoken-0.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.6 kB)\n", + "Requirement already satisfied: Pillow in /usr/local/lib/python3.11/dist-packages (from llama-models==0.1.0rc10->llama-stack==0.1.0rc10) (11.1.0)\n", + "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.11/dist-packages (from llama-stack-client==0.1.0rc10->llama-stack==0.1.0rc10) (3.7.1)\n", + "Requirement already satisfied: click in /usr/local/lib/python3.11/dist-packages (from llama-stack-client==0.1.0rc10->llama-stack==0.1.0rc10) (8.1.8)\n", + "Requirement already satisfied: distro<2,>=1.7.0 in /usr/local/lib/python3.11/dist-packages (from llama-stack-client==0.1.0rc10->llama-stack==0.1.0rc10) (1.9.0)\n", + "Requirement already satisfied: pandas in /usr/local/lib/python3.11/dist-packages (from llama-stack-client==0.1.0rc10->llama-stack==0.1.0rc10) (2.2.2)\n", + "Collecting pyaml (from llama-stack-client==0.1.0rc10->llama-stack==0.1.0rc10)\n", + " Downloading pyaml-25.1.0-py3-none-any.whl.metadata (12 kB)\n", + "Requirement already satisfied: sniffio in /usr/local/lib/python3.11/dist-packages (from llama-stack-client==0.1.0rc10->llama-stack==0.1.0rc10) (1.3.1)\n", + "Requirement already satisfied: tqdm in /usr/local/lib/python3.11/dist-packages (from llama-stack-client==0.1.0rc10->llama-stack==0.1.0rc10) (4.67.1)\n", + "Requirement already satisfied: typing-extensions<5,>=4.7 in /usr/local/lib/python3.11/dist-packages (from llama-stack-client==0.1.0rc10->llama-stack==0.1.0rc10) (4.12.2)\n", + "Requirement already satisfied: certifi in /usr/local/lib/python3.11/dist-packages (from httpx->llama-stack==0.1.0rc10) (2024.12.14)\n", + "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.11/dist-packages (from httpx->llama-stack==0.1.0rc10) (1.0.7)\n", + "Requirement already satisfied: idna in /usr/local/lib/python3.11/dist-packages (from httpx->llama-stack==0.1.0rc10) (3.10)\n", + "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.11/dist-packages (from httpcore==1.*->httpx->llama-stack==0.1.0rc10) (0.14.0)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.11/dist-packages (from pydantic>=2->llama-stack==0.1.0rc10) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.27.2 in /usr/local/lib/python3.11/dist-packages (from pydantic>=2->llama-stack==0.1.0rc10) (2.27.2)\n", + "Collecting pycryptodomex>=3.8 (from blobfile->llama-stack==0.1.0rc10)\n", + " Downloading pycryptodomex-3.21.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.4 kB)\n", + "Requirement already satisfied: urllib3<3,>=1.25.3 in /usr/local/lib/python3.11/dist-packages (from blobfile->llama-stack==0.1.0rc10) (2.3.0)\n", + "Requirement already satisfied: lxml>=4.9 in /usr/local/lib/python3.11/dist-packages (from blobfile->llama-stack==0.1.0rc10) (5.3.0)\n", + "Requirement already satisfied: filelock>=3.0 in /usr/local/lib/python3.11/dist-packages (from blobfile->llama-stack==0.1.0rc10) (3.16.1)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.11/dist-packages (from huggingface-hub->llama-stack==0.1.0rc10) (2024.10.0)\n", + "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.11/dist-packages (from huggingface-hub->llama-stack==0.1.0rc10) (24.2)\n", + "Requirement already satisfied: wcwidth in /usr/local/lib/python3.11/dist-packages (from prompt-toolkit->llama-stack==0.1.0rc10) (0.2.13)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests->llama-stack==0.1.0rc10) (3.4.1)\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.11/dist-packages (from rich->llama-stack==0.1.0rc10) (3.0.0)\n", + "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.11/dist-packages (from rich->llama-stack==0.1.0rc10) (2.18.0)\n", + "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.11/dist-packages (from markdown-it-py>=2.2.0->rich->llama-stack==0.1.0rc10) (0.1.2)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.11/dist-packages (from jinja2->llama-models==0.1.0rc10->llama-stack==0.1.0rc10) (3.0.2)\n", + "Requirement already satisfied: numpy>=1.23.2 in /usr/local/lib/python3.11/dist-packages (from pandas->llama-stack-client==0.1.0rc10->llama-stack==0.1.0rc10) (1.26.4)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/dist-packages (from pandas->llama-stack-client==0.1.0rc10->llama-stack==0.1.0rc10) (2.8.2)\n", + "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas->llama-stack-client==0.1.0rc10->llama-stack==0.1.0rc10) (2024.2)\n", + "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas->llama-stack-client==0.1.0rc10->llama-stack==0.1.0rc10) (2024.2)\n", + "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.11/dist-packages (from tiktoken->llama-models==0.1.0rc10->llama-stack==0.1.0rc10) (2024.11.6)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.11/dist-packages (from python-dateutil>=2.8.2->pandas->llama-stack-client==0.1.0rc10->llama-stack==0.1.0rc10) (1.17.0)\n", + "Downloading https://test-files.pythonhosted.org/packages/68/22/4a170fbe01095df81e76c7bf8f35c716c1a0a5ec4503da6e78695fab351c/llama_stack-0.1.0rc10-py3-none-any.whl (532 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m532.7/532.7 kB\u001b[0m \u001b[31m14.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading https://test-files.pythonhosted.org/packages/45/2b/6a6947d5915054b9980f82606942f1b79960a27168299254ca12e5b5795b/llama_models-0.1.0rc10-py3-none-any.whl (1.6 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m20.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading https://test-files.pythonhosted.org/packages/d6/85/a4fd621c4ae4db7339ab098b37bf4b4ad3cc12440e75ef10ec524e28ef7d/llama_stack_client-0.1.0rc10-py3-none-any.whl (328 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m328.5/328.5 kB\u001b[0m \u001b[31m29.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading blobfile-3.0.0-py3-none-any.whl (75 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.4/75.4 kB\u001b[0m \u001b[31m7.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading python_dotenv-1.0.1-py3-none-any.whl (19 kB)\n", + "Downloading pycryptodomex-3.21.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.3/2.3 MB\u001b[0m \u001b[31m57.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading pyaml-25.1.0-py3-none-any.whl (26 kB)\n", + "Downloading tiktoken-0.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m64.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hBuilding wheels for collected packages: fire\n", + " Building wheel for fire (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for fire: filename=fire-0.7.0-py3-none-any.whl size=114249 sha256=3a37285ecae37a5fb69bbad717aabdb8c13f0da7906668b7c123475eefa41c3b\n", + " Stored in directory: /root/.cache/pip/wheels/46/54/24/1624fd5b8674eb1188623f7e8e17cdf7c0f6c24b609dfb8a89\n", + "Successfully built fire\n", + "Installing collected packages: python-dotenv, pycryptodomex, pyaml, fire, tiktoken, blobfile, llama-stack-client, llama-models, llama-stack\n", + "Successfully installed blobfile-3.0.0 fire-0.7.0 llama-models-0.1.0rc10 llama-stack-0.1.0rc10 llama-stack-client-0.1.0rc10 pyaml-25.1.0 pycryptodomex-3.21.0 python-dotenv-1.0.1 tiktoken-0.8.0\n" ] } ], "source": [ + "# NBVAL_SKIP\n", + "\n", "!apt-get install -y bubblewrap\n", - "!pip install -U llama-stack" + "# install a branch of llama stack\n", + "!pip install llama-stack" ] }, { @@ -172,7 +218,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 2, "id": "HaepEZXCDgif", "metadata": { "colab": { @@ -180,198 +226,333 @@ }, "collapsed": true, "id": "HaepEZXCDgif", - "outputId": "9c268d26-7444-4741-f14d-3911eea8e4eb" + "outputId": "9314f698-593d-4c1a-ea15-15c735dc1023" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Requirement already satisfied: llama-stack in /usr/local/lib/python3.10/dist-packages (0.0.61)\r\n", - "Requirement already satisfied: blobfile in /usr/local/lib/python3.10/dist-packages (from llama-stack) (3.0.0)\r\n", - "Requirement already satisfied: fire in /usr/local/lib/python3.10/dist-packages (from llama-stack) (0.7.0)\r\n", - "Requirement already satisfied: httpx in /usr/local/lib/python3.10/dist-packages (from llama-stack) (0.28.1)\r\n", - "Requirement already satisfied: huggingface-hub in /usr/local/lib/python3.10/dist-packages (from llama-stack) (0.26.5)\r\n", - "Requirement already satisfied: llama-models>=0.0.61 in /usr/local/lib/python3.10/dist-packages (from llama-stack) (0.0.61)\r\n", - "Requirement already satisfied: llama-stack-client>=0.0.61 in /usr/local/lib/python3.10/dist-packages (from llama-stack) (0.0.61)\r\n", - "Requirement already satisfied: prompt-toolkit in /usr/local/lib/python3.10/dist-packages (from llama-stack) (3.0.48)\r\n", - "Requirement already satisfied: python-dotenv in /usr/local/lib/python3.10/dist-packages (from llama-stack) (1.0.1)\r\n", - "Requirement already satisfied: pydantic>=2 in /usr/local/lib/python3.10/dist-packages (from llama-stack) (2.10.3)\r\n", - "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from llama-stack) (2.32.3)\r\n", - "Requirement already satisfied: rich in /usr/local/lib/python3.10/dist-packages (from llama-stack) (13.9.4)\r\n", - "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from llama-stack) (75.1.0)\r\n", - "Requirement already satisfied: termcolor in /usr/local/lib/python3.10/dist-packages (from llama-stack) (2.5.0)\r\n", - "Requirement already satisfied: PyYAML in /usr/local/lib/python3.10/dist-packages (from llama-models>=0.0.61->llama-stack) (6.0.2)\r\n", - "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from llama-models>=0.0.61->llama-stack) (3.1.4)\r\n", - "Requirement already satisfied: tiktoken in /usr/local/lib/python3.10/dist-packages (from llama-models>=0.0.61->llama-stack) (0.8.0)\r\n", - "Requirement already satisfied: Pillow in /usr/local/lib/python3.10/dist-packages (from llama-models>=0.0.61->llama-stack) (10.4.0)\r\n", - "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.61->llama-stack) (3.7.1)\r\n", - "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.61->llama-stack) (8.1.7)\r\n", - "Requirement already satisfied: distro<2,>=1.7.0 in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.61->llama-stack) (1.9.0)\r\n", - "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.61->llama-stack) (2.2.2)\r\n", - "Requirement already satisfied: pyaml in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.61->llama-stack) (24.12.1)\r\n", - "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.61->llama-stack) (1.3.1)\r\n", - "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.61->llama-stack) (4.66.6)\r\n", - "Requirement already satisfied: typing-extensions<5,>=4.7 in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.61->llama-stack) (4.12.2)\r\n", - "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx->llama-stack) (2024.8.30)\r\n", - "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx->llama-stack) (1.0.7)\r\n", - "Requirement already satisfied: idna in /usr/local/lib/python3.10/dist-packages (from httpx->llama-stack) (3.10)\r\n", - "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx->llama-stack) (0.14.0)\r\n", - "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.10/dist-packages (from pydantic>=2->llama-stack) (0.7.0)\r\n", - "Requirement already satisfied: pydantic-core==2.27.1 in /usr/local/lib/python3.10/dist-packages (from pydantic>=2->llama-stack) (2.27.1)\r\n", - "Requirement already satisfied: pycryptodomex>=3.8 in /usr/local/lib/python3.10/dist-packages (from blobfile->llama-stack) (3.21.0)\r\n", - "Requirement already satisfied: urllib3<3,>=1.25.3 in /usr/local/lib/python3.10/dist-packages (from blobfile->llama-stack) (2.2.3)\r\n", - "Requirement already satisfied: lxml>=4.9 in /usr/local/lib/python3.10/dist-packages (from blobfile->llama-stack) (5.3.0)\r\n", - "Requirement already satisfied: filelock>=3.0 in /usr/local/lib/python3.10/dist-packages (from blobfile->llama-stack) (3.16.1)\n", - "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->llama-stack) (2024.9.0)\n", - "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->llama-stack) (24.2)\n", - "Requirement already satisfied: wcwidth in /usr/local/lib/python3.10/dist-packages (from prompt-toolkit->llama-stack) (0.2.13)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->llama-stack) (3.4.0)\n", - "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich->llama-stack) (3.0.0)\n", - "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich->llama-stack) (2.18.0)\n", - "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->llama-stack-client>=0.0.61->llama-stack) (1.2.2)\n", - "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich->llama-stack) (0.1.2)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->llama-models>=0.0.61->llama-stack) (3.0.2)\n", - "Requirement already satisfied: numpy>=1.22.4 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-stack-client>=0.0.61->llama-stack) (1.26.4)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-stack-client>=0.0.61->llama-stack) (2.8.2)\n", - "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-stack-client>=0.0.61->llama-stack) (2024.2)\n", - "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-stack-client>=0.0.61->llama-stack) (2024.2)\n", - "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken->llama-models>=0.0.61->llama-stack) (2024.9.11)\n", - "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas->llama-stack-client>=0.0.61->llama-stack) (1.17.0)\n", + "Requirement already satisfied: llama-stack in /usr/local/lib/python3.11/dist-packages (0.1.0rc10)\r\n", + "Requirement already satisfied: blobfile in /usr/local/lib/python3.11/dist-packages (from llama-stack) (3.0.0)\r\n", + "Requirement already satisfied: fire in /usr/local/lib/python3.11/dist-packages (from llama-stack) (0.7.0)\r\n", + "Requirement already satisfied: httpx in /usr/local/lib/python3.11/dist-packages (from llama-stack) (0.28.1)\r\n", + "Requirement already satisfied: huggingface-hub in /usr/local/lib/python3.11/dist-packages (from llama-stack) (0.27.1)\r\n", + "Requirement already satisfied: llama-models==0.1.0rc10 in /usr/local/lib/python3.11/dist-packages (from llama-stack) (0.1.0rc10)\r\n", + "Requirement already satisfied: llama-stack-client==0.1.0rc10 in /usr/local/lib/python3.11/dist-packages (from llama-stack) (0.1.0rc10)\r\n", + "Requirement already satisfied: prompt-toolkit in /usr/local/lib/python3.11/dist-packages (from llama-stack) (3.0.48)\r\n", + "Requirement already satisfied: python-dotenv in /usr/local/lib/python3.11/dist-packages (from llama-stack) (1.0.1)\r\n", + "Requirement already satisfied: pydantic>=2 in /usr/local/lib/python3.11/dist-packages (from llama-stack) (2.10.5)\r\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.11/dist-packages (from llama-stack) (2.32.3)\r\n", + "Requirement already satisfied: rich in /usr/local/lib/python3.11/dist-packages (from llama-stack) (13.9.4)\r\n", + "Requirement already satisfied: setuptools in /usr/local/lib/python3.11/dist-packages (from llama-stack) (75.1.0)\r\n", + "Requirement already satisfied: termcolor in /usr/local/lib/python3.11/dist-packages (from llama-stack) (2.5.0)\r\n", + "Requirement already satisfied: PyYAML in /usr/local/lib/python3.11/dist-packages (from llama-models==0.1.0rc10->llama-stack) (6.0.2)\r\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.11/dist-packages (from llama-models==0.1.0rc10->llama-stack) (3.1.5)\r\n", + "Requirement already satisfied: tiktoken in /usr/local/lib/python3.11/dist-packages (from llama-models==0.1.0rc10->llama-stack) (0.8.0)\r\n", + "Requirement already satisfied: Pillow in /usr/local/lib/python3.11/dist-packages (from llama-models==0.1.0rc10->llama-stack) (11.1.0)\r\n", + "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.11/dist-packages (from llama-stack-client==0.1.0rc10->llama-stack) (3.7.1)\r\n", + "Requirement already satisfied: click in /usr/local/lib/python3.11/dist-packages (from llama-stack-client==0.1.0rc10->llama-stack) (8.1.8)\r\n", + "Requirement already satisfied: distro<2,>=1.7.0 in /usr/local/lib/python3.11/dist-packages (from llama-stack-client==0.1.0rc10->llama-stack) (1.9.0)\r\n", + "Requirement already satisfied: pandas in /usr/local/lib/python3.11/dist-packages (from llama-stack-client==0.1.0rc10->llama-stack) (2.2.2)\r\n", + "Requirement already satisfied: pyaml in /usr/local/lib/python3.11/dist-packages (from llama-stack-client==0.1.0rc10->llama-stack) (25.1.0)\r\n", + "Requirement already satisfied: sniffio in /usr/local/lib/python3.11/dist-packages (from llama-stack-client==0.1.0rc10->llama-stack) (1.3.1)\r\n", + "Requirement already satisfied: tqdm in /usr/local/lib/python3.11/dist-packages (from llama-stack-client==0.1.0rc10->llama-stack) (4.67.1)\r\n", + "Requirement already satisfied: typing-extensions<5,>=4.7 in /usr/local/lib/python3.11/dist-packages (from llama-stack-client==0.1.0rc10->llama-stack) (4.12.2)\r\n", + "Requirement already satisfied: certifi in /usr/local/lib/python3.11/dist-packages (from httpx->llama-stack) (2024.12.14)\r\n", + "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.11/dist-packages (from httpx->llama-stack) (1.0.7)\r\n", + "Requirement already satisfied: idna in /usr/local/lib/python3.11/dist-packages (from httpx->llama-stack) (3.10)\r\n", + "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.11/dist-packages (from httpcore==1.*->httpx->llama-stack) (0.14.0)\r\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.11/dist-packages (from pydantic>=2->llama-stack) (0.7.0)\r\n", + "Requirement already satisfied: pydantic-core==2.27.2 in /usr/local/lib/python3.11/dist-packages (from pydantic>=2->llama-stack) (2.27.2)\r\n", + "Requirement already satisfied: pycryptodomex>=3.8 in /usr/local/lib/python3.11/dist-packages (from blobfile->llama-stack) (3.21.0)\r\n", + "Requirement already satisfied: urllib3<3,>=1.25.3 in /usr/local/lib/python3.11/dist-packages (from blobfile->llama-stack) (2.3.0)\r\n", + "Requirement already satisfied: lxml>=4.9 in /usr/local/lib/python3.11/dist-packages (from blobfile->llama-stack) (5.3.0)\r\n", + "Requirement already satisfied: filelock>=3.0 in /usr/local/lib/python3.11/dist-packages (from blobfile->llama-stack) (3.16.1)\r\n", + "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.11/dist-packages (from huggingface-hub->llama-stack) (2024.10.0)\r\n", + "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.11/dist-packages (from huggingface-hub->llama-stack) (24.2)\r\n", + "Requirement already satisfied: wcwidth in /usr/local/lib/python3.11/dist-packages (from prompt-toolkit->llama-stack) (0.2.13)\r\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests->llama-stack) (3.4.1)\r\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.11/dist-packages (from rich->llama-stack) (3.0.0)\r\n", + "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.11/dist-packages (from rich->llama-stack) (2.18.0)\n", + "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.11/dist-packages (from markdown-it-py>=2.2.0->rich->llama-stack) (0.1.2)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.11/dist-packages (from jinja2->llama-models==0.1.0rc10->llama-stack) (3.0.2)\n", + "Requirement already satisfied: numpy>=1.23.2 in /usr/local/lib/python3.11/dist-packages (from pandas->llama-stack-client==0.1.0rc10->llama-stack) (1.26.4)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/dist-packages (from pandas->llama-stack-client==0.1.0rc10->llama-stack) (2.8.2)\n", + "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas->llama-stack-client==0.1.0rc10->llama-stack) (2024.2)\n", + "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas->llama-stack-client==0.1.0rc10->llama-stack) (2024.2)\n", + "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.11/dist-packages (from tiktoken->llama-models==0.1.0rc10->llama-stack) (2024.11.6)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.11/dist-packages (from python-dateutil>=2.8.2->pandas->llama-stack-client==0.1.0rc10->llama-stack) (1.17.0)\n", "Installing pip dependencies\n", - "Requirement already satisfied: pillow in /usr/local/lib/python3.10/dist-packages (10.4.0)\n", - "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.46.3)\n", - "Requirement already satisfied: psycopg2-binary in /usr/local/lib/python3.10/dist-packages (2.9.10)\n", - "Requirement already satisfied: aiosqlite in /usr/local/lib/python3.10/dist-packages (0.20.0)\n", - "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (4.66.6)\n", - "Requirement already satisfied: pypdf in /usr/local/lib/python3.10/dist-packages (5.1.0)\n", - "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (1.26.4)\n", - "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (1.5.2)\n", - "Requirement already satisfied: redis in /usr/local/lib/python3.10/dist-packages (5.2.1)\n", - "Requirement already satisfied: opentelemetry-sdk in /usr/local/lib/python3.10/dist-packages (1.28.2)\n", - "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.10/dist-packages (0.2.0)\n", - "Requirement already satisfied: blobfile in /usr/local/lib/python3.10/dist-packages (3.0.0)\n", - "Requirement already satisfied: together in /usr/local/lib/python3.10/dist-packages (1.3.5)\n", - "Requirement already satisfied: openai in /usr/local/lib/python3.10/dist-packages (1.54.5)\n", - "Requirement already satisfied: faiss-cpu in /usr/local/lib/python3.10/dist-packages (1.9.0.post1)\n", - "Requirement already satisfied: autoevals in /usr/local/lib/python3.10/dist-packages (0.0.110)\n", - "Requirement already satisfied: chardet in /usr/local/lib/python3.10/dist-packages (5.2.0)\n", - "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (3.9.1)\n", - "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (2.2.2)\n", - "Requirement already satisfied: opentelemetry-exporter-otlp-proto-http in /usr/local/lib/python3.10/dist-packages (1.28.2)\n", - "Requirement already satisfied: datasets in /usr/local/lib/python3.10/dist-packages (3.2.0)\n", - "Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (3.8.0)\n", - "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (1.13.1)\n", - "Requirement already satisfied: chromadb-client in /usr/local/lib/python3.10/dist-packages (0.5.23)\n", - "Requirement already satisfied: fastapi in /usr/local/lib/python3.10/dist-packages (0.115.6)\n", - "Requirement already satisfied: fire in /usr/local/lib/python3.10/dist-packages (0.7.0)\n", - "Requirement already satisfied: httpx in /usr/local/lib/python3.10/dist-packages (0.28.1)\n", - "Requirement already satisfied: uvicorn in /usr/local/lib/python3.10/dist-packages (0.32.1)\n", - "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.16.1)\n", - "Requirement already satisfied: huggingface-hub<1.0,>=0.23.2 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.26.5)\n", - "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (24.2)\n", - "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.2)\n", - "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2024.9.11)\n", - "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.32.3)\n", - "Requirement already satisfied: tokenizers<0.21,>=0.20 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.20.3)\n", - "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.5)\n", - "Requirement already satisfied: typing_extensions>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiosqlite) (4.12.2)\n", - "Requirement already satisfied: joblib>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn) (1.4.2)\n", - "Requirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn) (3.5.0)\n", - "Requirement already satisfied: async-timeout>=4.0.3 in /usr/local/lib/python3.10/dist-packages (from redis) (4.0.3)\n", - "Requirement already satisfied: opentelemetry-api==1.28.2 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-sdk) (1.28.2)\n", - "Requirement already satisfied: opentelemetry-semantic-conventions==0.49b2 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-sdk) (0.49b2)\n", - "Requirement already satisfied: deprecated>=1.2.6 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-api==1.28.2->opentelemetry-sdk) (1.2.15)\n", - "Requirement already satisfied: importlib-metadata<=8.5.0,>=6.0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-api==1.28.2->opentelemetry-sdk) (8.5.0)\n", - "Requirement already satisfied: pycryptodomex>=3.8 in /usr/local/lib/python3.10/dist-packages (from blobfile) (3.21.0)\n", - "Requirement already satisfied: urllib3<3,>=1.25.3 in /usr/local/lib/python3.10/dist-packages (from blobfile) (2.2.3)\n", - "Requirement already satisfied: lxml>=4.9 in /usr/local/lib/python3.10/dist-packages (from blobfile) (5.3.0)\n", - "Requirement already satisfied: aiohttp<4.0.0,>=3.9.3 in /usr/local/lib/python3.10/dist-packages (from together) (3.11.10)\n", - "Requirement already satisfied: click<9.0.0,>=8.1.7 in /usr/local/lib/python3.10/dist-packages (from together) (8.1.7)\n", - "Requirement already satisfied: eval-type-backport<0.3.0,>=0.1.3 in /usr/local/lib/python3.10/dist-packages (from together) (0.2.0)\n", - "Requirement already satisfied: pyarrow>=10.0.1 in /usr/local/lib/python3.10/dist-packages (from together) (17.0.0)\n", - "Requirement already satisfied: pydantic<3.0.0,>=2.6.3 in /usr/local/lib/python3.10/dist-packages (from together) (2.10.3)\n", - "Requirement already satisfied: rich<14.0.0,>=13.8.1 in /usr/local/lib/python3.10/dist-packages (from together) (13.9.4)\n", - "Requirement already satisfied: tabulate<0.10.0,>=0.9.0 in /usr/local/lib/python3.10/dist-packages (from together) (0.9.0)\n", - "Requirement already satisfied: typer<0.14,>=0.9 in /usr/local/lib/python3.10/dist-packages (from together) (0.13.1)\n", - "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai) (3.7.1)\n", - "Requirement already satisfied: distro<2,>=1.7.0 in /usr/local/lib/python3.10/dist-packages (from openai) (1.9.0)\n", - "Requirement already satisfied: jiter<1,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from openai) (0.8.2)\n", - "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai) (1.3.1)\n", - "Requirement already satisfied: chevron in /usr/local/lib/python3.10/dist-packages (from autoevals) (0.14.0)\n", - "Requirement already satisfied: levenshtein in /usr/local/lib/python3.10/dist-packages (from autoevals) (0.26.1)\n", - "Requirement already satisfied: braintrust_core==0.0.54 in /usr/local/lib/python3.10/dist-packages (from autoevals) (0.0.54)\n", - "Requirement already satisfied: jsonschema in /usr/local/lib/python3.10/dist-packages (from autoevals) (4.23.0)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas) (2.8.2)\n", - "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas) (2024.2)\n", - "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas) (2024.2)\n", - "Requirement already satisfied: googleapis-common-protos~=1.52 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-exporter-otlp-proto-http) (1.66.0)\n", - "Requirement already satisfied: opentelemetry-exporter-otlp-proto-common==1.28.2 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-exporter-otlp-proto-http) (1.28.2)\n", - "Requirement already satisfied: opentelemetry-proto==1.28.2 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-exporter-otlp-proto-http) (1.28.2)\n", - "Requirement already satisfied: protobuf<6.0,>=5.0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-proto==1.28.2->opentelemetry-exporter-otlp-proto-http) (5.29.1)\n", - "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.3.8)\n", - "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets) (3.5.0)\n", - "Requirement already satisfied: multiprocess<0.70.17 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.70.16)\n", - "Requirement already satisfied: fsspec<=2024.9.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from fsspec[http]<=2024.9.0,>=2023.1.0->datasets) (2024.9.0)\n", - "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (1.3.1)\n", - "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (0.12.1)\n", - "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (4.55.3)\n", - "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (1.4.7)\n", - "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (3.2.0)\n", - "Requirement already satisfied: opentelemetry-exporter-otlp-proto-grpc>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from chromadb-client) (1.28.2)\n", - "Requirement already satisfied: overrides>=7.3.1 in /usr/local/lib/python3.10/dist-packages (from chromadb-client) (7.7.0)\n", - "Requirement already satisfied: posthog>=2.4.0 in /usr/local/lib/python3.10/dist-packages (from chromadb-client) (3.7.4)\n", - "Requirement already satisfied: tenacity>=8.2.3 in /usr/local/lib/python3.10/dist-packages (from chromadb-client) (9.0.0)\n", - "Requirement already satisfied: orjson>=3.9.12 in /usr/local/lib/python3.10/dist-packages (from chromadb-client) (3.10.12)\n", - "Requirement already satisfied: starlette<0.42.0,>=0.40.0 in /usr/local/lib/python3.10/dist-packages (from fastapi) (0.41.3)\n", - "Requirement already satisfied: termcolor in /usr/local/lib/python3.10/dist-packages (from fire) (2.5.0)\n", - "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx) (2024.8.30)\n", - "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx) (1.0.7)\n", - "Requirement already satisfied: idna in /usr/local/lib/python3.10/dist-packages (from httpx) (3.10)\n", - "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx) (0.14.0)\n", - "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.9.3->together) (2.4.4)\n", - "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.9.3->together) (1.3.1)\n", - "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.9.3->together) (24.2.0)\n", - "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.9.3->together) (1.5.0)\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.9.3->together) (6.1.0)\n", - "Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.9.3->together) (0.2.1)\n", - "Requirement already satisfied: yarl<2.0,>=1.17.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.9.3->together) (1.18.3)\n", - "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai) (1.2.2)\n", - "Requirement already satisfied: wrapt<2,>=1.10 in /usr/local/lib/python3.10/dist-packages (from deprecated>=1.2.6->opentelemetry-api==1.28.2->opentelemetry-sdk) (1.17.0)\n", - "Requirement already satisfied: grpcio<2.0.0,>=1.63.2 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb-client) (1.68.1)\n", - "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from posthog>=2.4.0->chromadb-client) (1.17.0)\n", - "Requirement already satisfied: monotonic>=1.5 in /usr/local/lib/python3.10/dist-packages (from posthog>=2.4.0->chromadb-client) (1.6)\n", - "Requirement already satisfied: backoff>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from posthog>=2.4.0->chromadb-client) (2.2.1)\n", - "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3.0.0,>=2.6.3->together) (0.7.0)\n", - "Requirement already satisfied: pydantic-core==2.27.1 in /usr/local/lib/python3.10/dist-packages (from pydantic<3.0.0,>=2.6.3->together) (2.27.1)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4.0)\n", - "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich<14.0.0,>=13.8.1->together) (3.0.0)\n", - "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich<14.0.0,>=13.8.1->together) (2.18.0)\n", - "Requirement already satisfied: shellingham>=1.3.0 in /usr/local/lib/python3.10/dist-packages (from typer<0.14,>=0.9->together) (1.5.4)\n", - "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.10/dist-packages (from jsonschema->autoevals) (2024.10.1)\n", - "Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.10/dist-packages (from jsonschema->autoevals) (0.35.1)\n", - "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from jsonschema->autoevals) (0.22.3)\n", - "Requirement already satisfied: rapidfuzz<4.0.0,>=3.9.0 in /usr/local/lib/python3.10/dist-packages (from levenshtein->autoevals) (3.10.1)\n", - "Requirement already satisfied: zipp>=3.20 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata<=8.5.0,>=6.0->opentelemetry-api==1.28.2->opentelemetry-sdk) (3.21.0)\n", - "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich<14.0.0,>=13.8.1->together) (0.1.2)\n", - "sentence-transformers --no-deps\n", - "Requirement already satisfied: sentence-transformers in /usr/local/lib/python3.10/dist-packages (3.2.1)\n", + "Requirement already satisfied: pandas in /usr/local/lib/python3.11/dist-packages (2.2.2)\n", + "Collecting together\n", + " Downloading together-1.3.11-py3-none-any.whl.metadata (11 kB)\n", + "Collecting datasets\n", + " Downloading datasets-3.2.0-py3-none-any.whl.metadata (20 kB)\n", + "Requirement already satisfied: transformers in /usr/local/lib/python3.11/dist-packages (4.47.1)\n", + "Requirement already satisfied: blobfile in /usr/local/lib/python3.11/dist-packages (3.0.0)\n", + "Requirement already satisfied: opentelemetry-sdk in /usr/local/lib/python3.11/dist-packages (1.29.0)\n", + "Collecting redis\n", + " Downloading redis-5.2.1-py3-none-any.whl.metadata (9.1 kB)\n", + "Requirement already satisfied: matplotlib in /usr/local/lib/python3.11/dist-packages (3.10.0)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.11/dist-packages (2.32.3)\n", + "Requirement already satisfied: chardet in /usr/local/lib/python3.11/dist-packages (5.2.0)\n", + "Collecting chromadb-client\n", + " Downloading chromadb_client-0.6.3-py3-none-any.whl.metadata (2.4 kB)\n", + "Collecting psycopg2-binary\n", + " Downloading psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.9 kB)\n", + "Collecting mcp\n", + " Downloading mcp-1.2.0-py3-none-any.whl.metadata (15 kB)\n", + "Requirement already satisfied: pillow in /usr/local/lib/python3.11/dist-packages (11.1.0)\n", + "Requirement already satisfied: scipy in /usr/local/lib/python3.11/dist-packages (1.13.1)\n", + "Requirement already satisfied: tqdm in /usr/local/lib/python3.11/dist-packages (4.67.1)\n", + "Requirement already satisfied: nltk in /usr/local/lib/python3.11/dist-packages (3.9.1)\n", + "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.11/dist-packages (0.2.0)\n", + "Collecting faiss-cpu\n", + " Downloading faiss_cpu-1.9.0.post1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.4 kB)\n", + "Collecting opentelemetry-exporter-otlp-proto-http\n", + " Downloading opentelemetry_exporter_otlp_proto_http-1.29.0-py3-none-any.whl.metadata (2.2 kB)\n", + "Collecting autoevals\n", + " Downloading autoevals-0.0.117-py3-none-any.whl.metadata (12 kB)\n", + "Collecting pypdf\n", + " Downloading pypdf-5.1.0-py3-none-any.whl.metadata (7.2 kB)\n", + "Collecting aiosqlite\n", + " Downloading aiosqlite-0.20.0-py3-none-any.whl.metadata (4.3 kB)\n", + "Requirement already satisfied: numpy in /usr/local/lib/python3.11/dist-packages (1.26.4)\n", + "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.11/dist-packages (1.6.0)\n", + "Requirement already satisfied: openai in /usr/local/lib/python3.11/dist-packages (1.59.6)\n", + "Collecting fastapi\n", + " Downloading fastapi-0.115.6-py3-none-any.whl.metadata (27 kB)\n", + "Requirement already satisfied: fire in /usr/local/lib/python3.11/dist-packages (0.7.0)\n", + "Requirement already satisfied: httpx in /usr/local/lib/python3.11/dist-packages (0.28.1)\n", + "Collecting uvicorn\n", + " Downloading uvicorn-0.34.0-py3-none-any.whl.metadata (6.5 kB)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/dist-packages (from pandas) (2.8.2)\n", + "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas) (2024.2)\n", + "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas) (2024.2)\n", + "Requirement already satisfied: aiohttp<4.0.0,>=3.9.3 in /usr/local/lib/python3.11/dist-packages (from together) (3.11.11)\n", + "Requirement already satisfied: click<9.0.0,>=8.1.7 in /usr/local/lib/python3.11/dist-packages (from together) (8.1.8)\n", + "Requirement already satisfied: eval-type-backport<0.3.0,>=0.1.3 in /usr/local/lib/python3.11/dist-packages (from together) (0.2.2)\n", + "Requirement already satisfied: filelock<4.0.0,>=3.13.1 in /usr/local/lib/python3.11/dist-packages (from together) (3.16.1)\n", + "Collecting pillow\n", + " Downloading pillow-10.4.0-cp311-cp311-manylinux_2_28_x86_64.whl.metadata (9.2 kB)\n", + "Requirement already satisfied: pyarrow>=10.0.1 in /usr/local/lib/python3.11/dist-packages (from together) (17.0.0)\n", + "Requirement already satisfied: pydantic<3.0.0,>=2.6.3 in /usr/local/lib/python3.11/dist-packages (from together) (2.10.5)\n", + "Requirement already satisfied: rich<14.0.0,>=13.8.1 in /usr/local/lib/python3.11/dist-packages (from together) (13.9.4)\n", + "Requirement already satisfied: tabulate<0.10.0,>=0.9.0 in /usr/local/lib/python3.11/dist-packages (from together) (0.9.0)\n", + "Requirement already satisfied: typer<0.16,>=0.9 in /usr/local/lib/python3.11/dist-packages (from together) (0.15.1)\n", + "Collecting dill<0.3.9,>=0.3.0 (from datasets)\n", + " Downloading dill-0.3.8-py3-none-any.whl.metadata (10 kB)\n", + "Collecting xxhash (from datasets)\n", + " Downloading xxhash-3.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (12 kB)\n", + "Collecting multiprocess<0.70.17 (from datasets)\n", + " Downloading multiprocess-0.70.16-py311-none-any.whl.metadata (7.2 kB)\n", + "Collecting fsspec<=2024.9.0,>=2023.1.0 (from fsspec[http]<=2024.9.0,>=2023.1.0->datasets)\n", + " Downloading fsspec-2024.9.0-py3-none-any.whl.metadata (11 kB)\n", + "Requirement already satisfied: huggingface-hub>=0.23.0 in /usr/local/lib/python3.11/dist-packages (from datasets) (0.27.1)\n", + "Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from datasets) (24.2)\n", + "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.11/dist-packages (from datasets) (6.0.2)\n", + "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.11/dist-packages (from transformers) (2024.11.6)\n", + "Requirement already satisfied: tokenizers<0.22,>=0.21 in /usr/local/lib/python3.11/dist-packages (from transformers) (0.21.0)\n", + "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.11/dist-packages (from transformers) (0.5.2)\n", + "Requirement already satisfied: pycryptodomex>=3.8 in /usr/local/lib/python3.11/dist-packages (from blobfile) (3.21.0)\n", + "Requirement already satisfied: urllib3<3,>=1.25.3 in /usr/local/lib/python3.11/dist-packages (from blobfile) (2.3.0)\n", + "Requirement already satisfied: lxml>=4.9 in /usr/local/lib/python3.11/dist-packages (from blobfile) (5.3.0)\n", + "Requirement already satisfied: opentelemetry-api==1.29.0 in /usr/local/lib/python3.11/dist-packages (from opentelemetry-sdk) (1.29.0)\n", + "Requirement already satisfied: opentelemetry-semantic-conventions==0.50b0 in /usr/local/lib/python3.11/dist-packages (from opentelemetry-sdk) (0.50b0)\n", + "Requirement already satisfied: typing-extensions>=3.7.4 in /usr/local/lib/python3.11/dist-packages (from opentelemetry-sdk) (4.12.2)\n", + "Requirement already satisfied: deprecated>=1.2.6 in /usr/local/lib/python3.11/dist-packages (from opentelemetry-api==1.29.0->opentelemetry-sdk) (1.2.15)\n", + "Requirement already satisfied: importlib-metadata<=8.5.0,>=6.0 in /usr/local/lib/python3.11/dist-packages (from opentelemetry-api==1.29.0->opentelemetry-sdk) (8.5.0)\n", + "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (1.3.1)\n", + "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (0.12.1)\n", + "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (4.55.3)\n", + "Requirement already satisfied: kiwisolver>=1.3.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (1.4.8)\n", + "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (3.2.1)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests) (3.4.1)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/dist-packages (from requests) (3.10)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/dist-packages (from requests) (2024.12.14)\n", + "Collecting opentelemetry-exporter-otlp-proto-grpc>=1.2.0 (from chromadb-client)\n", + " Downloading opentelemetry_exporter_otlp_proto_grpc-1.29.0-py3-none-any.whl.metadata (2.2 kB)\n", + "Collecting overrides>=7.3.1 (from chromadb-client)\n", + " Downloading overrides-7.7.0-py3-none-any.whl.metadata (5.8 kB)\n", + "Collecting posthog>=2.4.0 (from chromadb-client)\n", + " Downloading posthog-3.8.4-py2.py3-none-any.whl.metadata (2.8 kB)\n", + "Requirement already satisfied: tenacity>=8.2.3 in /usr/local/lib/python3.11/dist-packages (from chromadb-client) (9.0.0)\n", + "Requirement already satisfied: orjson>=3.9.12 in /usr/local/lib/python3.11/dist-packages (from chromadb-client) (3.10.14)\n", + "Collecting anyio>=4.5 (from mcp)\n", + " Downloading anyio-4.8.0-py3-none-any.whl.metadata (4.6 kB)\n", + "Collecting httpx-sse>=0.4 (from mcp)\n", + " Downloading httpx_sse-0.4.0-py3-none-any.whl.metadata (9.0 kB)\n", + "Collecting pydantic-settings>=2.6.1 (from mcp)\n", + " Downloading pydantic_settings-2.7.1-py3-none-any.whl.metadata (3.5 kB)\n", + "Collecting sse-starlette>=1.6.1 (from mcp)\n", + " Downloading sse_starlette-2.2.1-py3-none-any.whl.metadata (7.8 kB)\n", + "Collecting starlette>=0.27 (from mcp)\n", + " Downloading starlette-0.45.2-py3-none-any.whl.metadata (6.3 kB)\n", + "Requirement already satisfied: joblib in /usr/local/lib/python3.11/dist-packages (from nltk) (1.4.2)\n", + "Requirement already satisfied: googleapis-common-protos~=1.52 in /usr/local/lib/python3.11/dist-packages (from opentelemetry-exporter-otlp-proto-http) (1.66.0)\n", + "Collecting opentelemetry-exporter-otlp-proto-common==1.29.0 (from opentelemetry-exporter-otlp-proto-http)\n", + " Downloading opentelemetry_exporter_otlp_proto_common-1.29.0-py3-none-any.whl.metadata (1.8 kB)\n", + "Collecting opentelemetry-proto==1.29.0 (from opentelemetry-exporter-otlp-proto-http)\n", + " Downloading opentelemetry_proto-1.29.0-py3-none-any.whl.metadata (2.3 kB)\n", + "Collecting protobuf<6.0,>=5.0 (from opentelemetry-proto==1.29.0->opentelemetry-exporter-otlp-proto-http)\n", + " Downloading protobuf-5.29.3-cp38-abi3-manylinux2014_x86_64.whl.metadata (592 bytes)\n", + "Collecting chevron (from autoevals)\n", + " Downloading chevron-0.14.0-py3-none-any.whl.metadata (4.9 kB)\n", + "Collecting levenshtein (from autoevals)\n", + " Downloading levenshtein-0.26.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.2 kB)\n", + "Collecting braintrust_core==0.0.58 (from autoevals)\n", + " Downloading braintrust_core-0.0.58-py3-none-any.whl.metadata (669 bytes)\n", + "Requirement already satisfied: jsonschema in /usr/local/lib/python3.11/dist-packages (from autoevals) (4.23.0)\n", + "Requirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.11/dist-packages (from scikit-learn) (3.5.0)\n", + "Requirement already satisfied: distro<2,>=1.7.0 in /usr/local/lib/python3.11/dist-packages (from openai) (1.9.0)\n", + "Requirement already satisfied: jiter<1,>=0.4.0 in /usr/local/lib/python3.11/dist-packages (from openai) (0.8.2)\n", + "Requirement already satisfied: sniffio in /usr/local/lib/python3.11/dist-packages (from openai) (1.3.1)\n", + "Collecting starlette>=0.27 (from mcp)\n", + " Downloading starlette-0.41.3-py3-none-any.whl.metadata (6.0 kB)\n", + "Requirement already satisfied: termcolor in /usr/local/lib/python3.11/dist-packages (from fire) (2.5.0)\n", + "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.11/dist-packages (from httpx) (1.0.7)\n", + "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.11/dist-packages (from httpcore==1.*->httpx) (0.14.0)\n", + "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp<4.0.0,>=3.9.3->together) (2.4.4)\n", + "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.11/dist-packages (from aiohttp<4.0.0,>=3.9.3->together) (1.3.2)\n", + "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp<4.0.0,>=3.9.3->together) (24.3.0)\n", + "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.11/dist-packages (from aiohttp<4.0.0,>=3.9.3->together) (1.5.0)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.11/dist-packages (from aiohttp<4.0.0,>=3.9.3->together) (6.1.0)\n", + "Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp<4.0.0,>=3.9.3->together) (0.2.1)\n", + "Requirement already satisfied: yarl<2.0,>=1.17.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp<4.0.0,>=3.9.3->together) (1.18.3)\n", + "Requirement already satisfied: wrapt<2,>=1.10 in /usr/local/lib/python3.11/dist-packages (from deprecated>=1.2.6->opentelemetry-api==1.29.0->opentelemetry-sdk) (1.17.0)\n", + "Requirement already satisfied: grpcio<2.0.0,>=1.63.2 in /usr/local/lib/python3.11/dist-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb-client) (1.69.0)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.11/dist-packages (from posthog>=2.4.0->chromadb-client) (1.17.0)\n", + "Collecting monotonic>=1.5 (from posthog>=2.4.0->chromadb-client)\n", + " Downloading monotonic-1.6-py2.py3-none-any.whl.metadata (1.5 kB)\n", + "Collecting backoff>=1.10.0 (from posthog>=2.4.0->chromadb-client)\n", + " Downloading backoff-2.2.1-py3-none-any.whl.metadata (14 kB)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.11/dist-packages (from pydantic<3.0.0,>=2.6.3->together) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.27.2 in /usr/local/lib/python3.11/dist-packages (from pydantic<3.0.0,>=2.6.3->together) (2.27.2)\n", + "Requirement already satisfied: python-dotenv>=0.21.0 in /usr/local/lib/python3.11/dist-packages (from pydantic-settings>=2.6.1->mcp) (1.0.1)\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.11/dist-packages (from rich<14.0.0,>=13.8.1->together) (3.0.0)\n", + "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.11/dist-packages (from rich<14.0.0,>=13.8.1->together) (2.18.0)\n", + "Requirement already satisfied: shellingham>=1.3.0 in /usr/local/lib/python3.11/dist-packages (from typer<0.16,>=0.9->together) (1.5.4)\n", + "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.11/dist-packages (from jsonschema->autoevals) (2024.10.1)\n", + "Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.11/dist-packages (from jsonschema->autoevals) (0.35.1)\n", + "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.11/dist-packages (from jsonschema->autoevals) (0.22.3)\n", + "Collecting rapidfuzz<4.0.0,>=3.9.0 (from levenshtein->autoevals)\n", + " Downloading rapidfuzz-3.11.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (11 kB)\n", + "Requirement already satisfied: zipp>=3.20 in /usr/local/lib/python3.11/dist-packages (from importlib-metadata<=8.5.0,>=6.0->opentelemetry-api==1.29.0->opentelemetry-sdk) (3.21.0)\n", + "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.11/dist-packages (from markdown-it-py>=2.2.0->rich<14.0.0,>=13.8.1->together) (0.1.2)\n", + "Downloading together-1.3.11-py3-none-any.whl (70 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m70.6/70.6 kB\u001b[0m \u001b[31m7.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading datasets-3.2.0-py3-none-any.whl (480 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m480.6/480.6 kB\u001b[0m \u001b[31m20.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading redis-5.2.1-py3-none-any.whl (261 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m261.5/261.5 kB\u001b[0m \u001b[31m25.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading chromadb_client-0.6.3-py3-none-any.whl (609 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m609.2/609.2 kB\u001b[0m \u001b[31m38.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading psycopg2_binary-2.9.10-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.0 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.0/3.0 MB\u001b[0m \u001b[31m100.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading mcp-1.2.0-py3-none-any.whl (66 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m66.5/66.5 kB\u001b[0m \u001b[31m7.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading pillow-10.4.0-cp311-cp311-manylinux_2_28_x86_64.whl (4.5 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.5/4.5 MB\u001b[0m \u001b[31m106.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading faiss_cpu-1.9.0.post1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (27.5 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m27.5/27.5 MB\u001b[0m \u001b[31m78.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading opentelemetry_exporter_otlp_proto_http-1.29.0-py3-none-any.whl (17 kB)\n", + "Downloading opentelemetry_exporter_otlp_proto_common-1.29.0-py3-none-any.whl (18 kB)\n", + "Downloading opentelemetry_proto-1.29.0-py3-none-any.whl (55 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m55.8/55.8 kB\u001b[0m \u001b[31m4.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading autoevals-0.0.117-py3-none-any.whl (41 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.4/41.4 kB\u001b[0m \u001b[31m4.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading braintrust_core-0.0.58-py3-none-any.whl (4.4 kB)\n", + "Downloading pypdf-5.1.0-py3-none-any.whl (297 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m298.0/298.0 kB\u001b[0m \u001b[31m24.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading aiosqlite-0.20.0-py3-none-any.whl (15 kB)\n", + "Downloading fastapi-0.115.6-py3-none-any.whl (94 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m94.8/94.8 kB\u001b[0m \u001b[31m9.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading uvicorn-0.34.0-py3-none-any.whl (62 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.3/62.3 kB\u001b[0m \u001b[31m5.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading anyio-4.8.0-py3-none-any.whl (96 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m96.0/96.0 kB\u001b[0m \u001b[31m9.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading dill-0.3.8-py3-none-any.whl (116 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m12.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading fsspec-2024.9.0-py3-none-any.whl (179 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m179.3/179.3 kB\u001b[0m \u001b[31m17.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading httpx_sse-0.4.0-py3-none-any.whl (7.8 kB)\n", + "Downloading multiprocess-0.70.16-py311-none-any.whl (143 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.5/143.5 kB\u001b[0m \u001b[31m14.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading opentelemetry_exporter_otlp_proto_grpc-1.29.0-py3-none-any.whl (18 kB)\n", + "Downloading overrides-7.7.0-py3-none-any.whl (17 kB)\n", + "Downloading posthog-3.8.4-py2.py3-none-any.whl (69 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m69.8/69.8 kB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading pydantic_settings-2.7.1-py3-none-any.whl (29 kB)\n", + "Downloading sse_starlette-2.2.1-py3-none-any.whl (10 kB)\n", + "Downloading starlette-0.41.3-py3-none-any.whl (73 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m73.2/73.2 kB\u001b[0m \u001b[31m7.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading chevron-0.14.0-py3-none-any.whl (11 kB)\n", + "Downloading levenshtein-0.26.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (162 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m162.7/162.7 kB\u001b[0m \u001b[31m17.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading xxhash-3.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.8/194.8 kB\u001b[0m \u001b[31m21.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading backoff-2.2.1-py3-none-any.whl (15 kB)\n", + "Downloading monotonic-1.6-py2.py3-none-any.whl (8.2 kB)\n", + "Downloading protobuf-5.29.3-cp38-abi3-manylinux2014_x86_64.whl (319 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m319.7/319.7 kB\u001b[0m \u001b[31m28.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading rapidfuzz-3.11.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.1 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m84.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hInstalling collected packages: monotonic, chevron, xxhash, uvicorn, redis, rapidfuzz, pypdf, psycopg2-binary, protobuf, pillow, overrides, httpx-sse, fsspec, faiss-cpu, dill, braintrust_core, backoff, anyio, aiosqlite, starlette, posthog, opentelemetry-proto, multiprocess, levenshtein, sse-starlette, pydantic-settings, opentelemetry-exporter-otlp-proto-common, fastapi, together, mcp, datasets, autoevals, opentelemetry-exporter-otlp-proto-http, opentelemetry-exporter-otlp-proto-grpc, chromadb-client\n", + " Attempting uninstall: protobuf\n", + " Found existing installation: protobuf 4.25.5\n", + " Uninstalling protobuf-4.25.5:\n", + " Successfully uninstalled protobuf-4.25.5\n", + " Attempting uninstall: pillow\n", + " Found existing installation: pillow 11.1.0\n", + " Uninstalling pillow-11.1.0:\n", + " Successfully uninstalled pillow-11.1.0\n", + " Attempting uninstall: fsspec\n", + " Found existing installation: fsspec 2024.10.0\n", + " Uninstalling fsspec-2024.10.0:\n", + " Successfully uninstalled fsspec-2024.10.0\n", + " Attempting uninstall: anyio\n", + " Found existing installation: anyio 3.7.1\n", + " Uninstalling anyio-3.7.1:\n", + " Successfully uninstalled anyio-3.7.1\n", + "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", + "jupyter-server 1.24.0 requires anyio<4,>=3.1.0, but you have anyio 4.8.0 which is incompatible.\n", + "gcsfs 2024.10.0 requires fsspec==2024.10.0, but you have fsspec 2024.9.0 which is incompatible.\n", + "tensorflow 2.17.1 requires protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3, but you have protobuf 5.29.3 which is incompatible.\u001b[0m\u001b[31m\n", + "\u001b[0mSuccessfully installed aiosqlite-0.20.0 anyio-4.8.0 autoevals-0.0.117 backoff-2.2.1 braintrust_core-0.0.58 chevron-0.14.0 chromadb-client-0.6.3 datasets-3.2.0 dill-0.3.8 faiss-cpu-1.9.0.post1 fastapi-0.115.6 fsspec-2024.9.0 httpx-sse-0.4.0 levenshtein-0.26.1 mcp-1.2.0 monotonic-1.6 multiprocess-0.70.16 opentelemetry-exporter-otlp-proto-common-1.29.0 opentelemetry-exporter-otlp-proto-grpc-1.29.0 opentelemetry-exporter-otlp-proto-http-1.29.0 opentelemetry-proto-1.29.0 overrides-7.7.0 pillow-10.4.0 posthog-3.8.4 protobuf-5.29.3 psycopg2-binary-2.9.10 pydantic-settings-2.7.1 pypdf-5.1.0 rapidfuzz-3.11.0 redis-5.2.1 sse-starlette-2.2.1 starlette-0.41.3 together-1.3.11 uvicorn-0.34.0 xxhash-3.5.0\n", "torch --index-url https://download.pytorch.org/whl/cpu\n", "Looking in indexes: https://download.pytorch.org/whl/cpu\n", - "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (2.5.1+cu121)\n", - "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch) (3.16.1)\n", - "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch) (4.12.2)\n", - "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch) (3.4.2)\n", - "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch) (3.1.4)\n", - "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch) (2024.9.0)\n", - "Requirement already satisfied: sympy==1.13.1 in /usr/local/lib/python3.10/dist-packages (from torch) (1.13.1)\n", - "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy==1.13.1->torch) (1.3.0)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch) (3.0.2)\n", + "Requirement already satisfied: torch in /usr/local/lib/python3.11/dist-packages (2.5.1+cu121)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.11/dist-packages (from torch) (3.16.1)\n", + "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.11/dist-packages (from torch) (4.12.2)\n", + "Requirement already satisfied: networkx in /usr/local/lib/python3.11/dist-packages (from torch) (3.4.2)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.11/dist-packages (from torch) (3.1.5)\n", + "Requirement already satisfied: fsspec in /usr/local/lib/python3.11/dist-packages (from torch) (2024.9.0)\n", + "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.11/dist-packages (from torch) (12.1.105)\n", + "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.11/dist-packages (from torch) (12.1.105)\n", + "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.11/dist-packages (from torch) (12.1.105)\n", + "Requirement already satisfied: nvidia-cudnn-cu12==9.1.0.70 in /usr/local/lib/python3.11/dist-packages (from torch) (9.1.0.70)\n", + "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.11/dist-packages (from torch) (12.1.3.1)\n", + "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.11/dist-packages (from torch) (11.0.2.54)\n", + "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.11/dist-packages (from torch) (10.3.2.106)\n", + "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.11/dist-packages (from torch) (11.4.5.107)\n", + "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.11/dist-packages (from torch) (12.1.0.106)\n", + "Requirement already satisfied: nvidia-nccl-cu12==2.21.5 in /usr/local/lib/python3.11/dist-packages (from torch) (2.21.5)\n", + "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.11/dist-packages (from torch) (12.1.105)\n", + "Requirement already satisfied: triton==3.1.0 in /usr/local/lib/python3.11/dist-packages (from torch) (3.1.0)\n", + "Requirement already satisfied: sympy==1.13.1 in /usr/local/lib/python3.11/dist-packages (from torch) (1.13.1)\n", + "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.11/dist-packages (from nvidia-cusolver-cu12==11.4.5.107->torch) (12.6.85)\n", + "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.11/dist-packages (from sympy==1.13.1->torch) (1.3.0)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.11/dist-packages (from jinja2->torch) (3.0.2)\n", + "sentence-transformers --no-deps\n", + "Requirement already satisfied: sentence-transformers in /usr/local/lib/python3.11/dist-packages (3.3.1)\n", "\u001b[32mBuild Successful!\u001b[0m\n" ] } ], "source": [ + "# NBVAL_SKIP\n", + "\n", "# This will build all the dependencies you will need\n", "!llama stack build --template together --image-type venv" ] @@ -390,68 +571,155 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 2, "id": "E1UFuJC570Tk", "metadata": { "colab": { "base_uri": "https://localhost:8080/", - "height": 1000 + "height": 1000, + "referenced_widgets": [ + "75307e3dee604d30aa44713e6e293e64", + "5ce87402a79342af995df41ac3940d55", + "fbbcc19886cc43b38424fbb184162c61", + "29212208db6b432eb4f708cd64258954", + "50dd8994a4cf486ebbec5ffd4322992a", + "f9b768c703494dd198f2978aff4892e8", + "1231b9e4cab34c33a38bee63543f1e75", + "754deb3970604d48a522bc9f021ad945", + "f6ecca7a1a8340fbbe056235a2714fc3", + "ef4f63fe9d8f4683a9d20becb6e4e2cb", + "7508f10c13634e7aa682cfb29c48d9e7", + "26f1430ca7cb4ad5b1b8df1ffdbd32a9", + "7cd2d9c9ea7b4d70902ffaff33033078", + "101288236cff40b8bb9dbad80dbbc7ee", + "d5c9977838a249eeab6ef628279b8155", + "d032d1e7b4b54ba28ac83c1a12b23876", + "321fce57c158432abeae496ae8a947aa", + "3ebe00201bdb4e119e3b74f684a58345", + "0f8bab6b8ed04774b386fe952aae66f1", + "cfcb6e456c354d99be91f161552f3376", + "61bd0d490c0e4c04a331cf9ce6b7d38f", + "7d8653fca29f4df3a7487733ff9db60b", + "943f8fcb66614353a51f32f8344b6122", + "0e695245b97c4bbc85e349fda3dc07b9", + "bb0d168c41f540b8ae42239d3938483a", + "87700a80125348f28c4f249bdf8b0a8d", + "8902c3622da540e496ed5b1524bd01ca", + "90432ec1c24b4607a935c94e130cd68d", + "464147b149824f20afc727751a702fc7", + "67e37a088be64a2ba786ca923b1017dd", + "98786f52ef5345b0b9164b9c1f2b8e18", + "0e1b9910a77d4b7fa69cb8926e6547d7", + "0b276315be4345be83da1e03905c8495", + "e11f8c3891284e07bd2572257afd5e1b", + "ee18d96394994d01b49d5b03b3d9a019", + "844b06df5749441fab6f61656ce581a9", + "e1c6b9a20e074f17aeba976b24e80c65", + "c690da8daa1e4f9ea73bcacdd92e8a6d", + "d0b161ae25c441e8b3caf7a3d88c1b05", + "47cf4b6b835d43388576a2abf4cc54f8", + "03bbebd659e64b5d9c29a73570c34854", + "b68e5097d2504d2cbd7e19aa1aac3a04", + "22a665deff88477b9372c0350c4c572b", + "5e535ed2b83e496ab57b1c80b615ab0c", + "d9de065c7f81443e98ddf066c7b5bd54", + "1e836106837c4ac7a11b36e700c46b64", + "55591e8179084fcfa3a61c8bd8d09dcb", + "de1ef93c41364eda9b4b111231057348", + "23b0b2f4f82c4a21846e91d7cea91da5", + "9e4d0fbb51284a7487c495c7b95a293d", + "b0f8cf1f79e04b5fb47a810f2c81bd7e", + "0c359bc4c94c46acbc9094354a15c33d", + "59d0b59b6c2248508d0601ff13878d33", + "891cb726d45c4fef8f2c74a56df5532b", + "fa39189070334939aea5fa4a7de5ec8b", + "f0e107dd6d54483aa367da0e337a97cd", + "861a00796f55470e85d94733eeee9a5f", + "5459633eb6e94ec391d13fcf67425726", + "b7b7467ece304ffbbd352b9b96a03aad", + "9dece059f1204e29b106fca9e191ddb3", + "e2e49c25d6fc4592b317e94cfabc2e5e", + "76d37a48a73946bab2821f097cf2605f", + "8e81ae00681347cb906b392c3656a64a", + "74bedc38b7da4e8a83b0c892d7aa59b5", + "d1e67c28b4664e8098dce8f5e80b8779", + "abe6cf39b784436993fcbe92221c31a3", + "d021a18ab70b4c7e8aec43932a124c36", + "72e7c092fb054b7ea0dcd2782b5d8a7d", + "8b1ea80221174fae943d5c9f997dfb57", + "f8073d625f80415dbf712cee434f6e3a", + "5f6014ba13fa4a659b9eb1b5f83599a7", + "327ff8f5292d47afbfebd3beea187739", + "988cac4341b646079fc73719f3f88ad7", + "900a4dac08f540dfb35c29f63236a12c", + "1e6009b9b0684b8fbaa379ea96f111ee", + "541b9b4e74614e2cb855bb90f03df538", + "ff256b2275f740ed82bca4f43b4d6fd2", + "3703041a499c426bb427ee008c81cde5", + "4b22bbacb995425fb32a2368f3685a92", + "49a66eeb9ef74de5ab8904fd90eb7558", + "08f9d125018b41c582a0fa1e234315f9", + "736c770230644894b85dbc34bd8f1d52", + "b67cbbf32f844a19b219be612d5038c9", + "774b513d64524ac7823a2cf13efa8d41", + "1e56da93bcf64ff490416d2b66cd3dc0", + "b7e35038ce344110b785753b655130f5", + "5472af91737446f4a4a2d92a3f684a45", + "9fb4368802da4a5a8101ba200d98403a", + "2e713bcc372e48b2a006558db4d1df68", + "1a277abd5ea44253bc6894bef258b52b", + "b3eedd82e7da4ce8b3ded70e49a2afd0", + "6f5c18cb8002471f8b3764effee37324", + "3bebac362b344e8d9103c5011613f1ea", + "670905a55b19458da69f83c8bcd511d1", + "ff54451a48394faaaa9d8cdb690d0718", + "36b5bc19b2d0407f8ab28ff0da2ce12d", + "879e48d9a9e04183903d94ffe98313d2", + "abce503d70594c2ca9afdc47847c125b", + "028e291ee53947bbbbc4bfb68c695f5f", + "a530662719374c95a9bef12e59e28c85", + "bffc0f4b12f141398535990709fd4f2c", + "04804c74e1dd43449d5f758cf5d0ba5e", + "95a506c3007c4525b01ee4e1600d671b", + "a0d6b0caeb2340fe96c8f5569e3d3ae4", + "30798f87a8b848d783fdacd71af5dc04", + "07ce54c75e76488ba4019a20b3707061", + "f023175de68445f98a6b01bb40ccdc6d", + "7389b79a0ff44cd68c7866995d728023", + "8e2b70ffe4eb4974bd6393fcc1292267", + "13eee164dc534424acb9dc9ee37a9465", + "722a7fe16af3422585a20c651345cfa4", + "f5596c1c9c4d42f3bc171961f9582eff", + "85d66e615b5742e78657b1e60c75fc72", + "731c02dc5dd446c3b22765575148e256", + "254ce460ce244c99a5afe39d5d51f6b7", + "4cf1dc345ace4da59f978f661487f975", + "8f30fca71bf24e5ca26e17c2321f893c", + "dd85d37dd1d14c7ea4592f8e11b2d2c8", + "3cb06377e4454f009d6b2aa7aa6ff0a9", + "4502477db4d948e693012364c2dcb370", + "52fe404ec9c14db2a7279b4c154eef3d" + ] }, "collapsed": true, "id": "E1UFuJC570Tk", - "outputId": "bac7c9ec-ad49-4040-af43-8869f0afe5ac" + "outputId": "aebb69d4-c167-4de5-eb8a-dd19dd538f63" }, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Not in Google Colab environment\n", + "\u001b[33mWarning: `bwrap` is not available. Code interpreter tool will not work correctly.\u001b[0m\n" + ] + }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO:llama_stack.distribution.resolver:Resolved 24 providers\n", - "INFO:llama_stack.distribution.resolver: inner-inference => together\n", - "INFO:llama_stack.distribution.resolver: inner-memory => faiss\n", - "INFO:llama_stack.distribution.resolver: models => __routing_table__\n", - "INFO:llama_stack.distribution.resolver: inference => __autorouted__\n", - "INFO:llama_stack.distribution.resolver: inner-safety => llama-guard\n", - "INFO:llama_stack.distribution.resolver: shields => __routing_table__\n", - "INFO:llama_stack.distribution.resolver: safety => __autorouted__\n", - "INFO:llama_stack.distribution.resolver: memory_banks => __routing_table__\n", - "INFO:llama_stack.distribution.resolver: memory => __autorouted__\n", - "INFO:llama_stack.distribution.resolver: agents => meta-reference\n", - "INFO:llama_stack.distribution.resolver: inner-datasetio => huggingface\n", - "INFO:llama_stack.distribution.resolver: inner-datasetio => localfs\n", - "INFO:llama_stack.distribution.resolver: datasets => __routing_table__\n", - "INFO:llama_stack.distribution.resolver: datasetio => __autorouted__\n", - "INFO:llama_stack.distribution.resolver: telemetry => meta-reference\n", - "INFO:llama_stack.distribution.resolver: inner-scoring => basic\n", - "INFO:llama_stack.distribution.resolver: inner-scoring => llm-as-judge\n", - "INFO:llama_stack.distribution.resolver: inner-scoring => braintrust\n", - "INFO:llama_stack.distribution.resolver: scoring_functions => __routing_table__\n", - "INFO:llama_stack.distribution.resolver: scoring => __autorouted__\n", - "INFO:llama_stack.distribution.resolver: inner-eval => meta-reference\n", - "INFO:llama_stack.distribution.resolver: eval_tasks => __routing_table__\n", - "INFO:llama_stack.distribution.resolver: eval => __autorouted__\n", - "INFO:llama_stack.distribution.resolver: inspect => __builtin__\n", - "INFO:llama_stack.distribution.resolver:\n", - "WARNING:opentelemetry.trace:Overriding of current TracerProvider is not allowed\n", - "INFO:llama_stack.distribution.stack:Models: meta-llama/Llama-3.1-405B-Instruct-FP8 served by together\n", - "INFO:llama_stack.distribution.stack:Models: meta-llama/Llama-3.1-70B-Instruct served by together\n", - "INFO:llama_stack.distribution.stack:Models: meta-llama/Llama-3.1-8B-Instruct served by together\n", - "INFO:llama_stack.distribution.stack:Models: meta-llama/Llama-3.2-11B-Vision-Instruct served by together\n", - "INFO:llama_stack.distribution.stack:Models: meta-llama/Llama-3.2-3B-Instruct served by together\n", - "INFO:llama_stack.distribution.stack:Models: meta-llama/Llama-3.2-90B-Vision-Instruct served by together\n", - "INFO:llama_stack.distribution.stack:Models: meta-llama/Llama-Guard-3-11B-Vision served by together\n", - "INFO:llama_stack.distribution.stack:Models: meta-llama/Llama-Guard-3-8B served by together\n", - "INFO:llama_stack.distribution.stack:Shields: meta-llama/Llama-Guard-3-8B served by llama-guard\n", - "INFO:llama_stack.distribution.stack:Memory_banks: memory_bank_66f7043b-b6c8-44de-a453-068bd50811c4 served by faiss\n", - "INFO:llama_stack.distribution.stack:Memory_banks: memory_bank_edf0d763-95bc-40d3-93a7-95b517162cfb served by faiss\n", - "INFO:llama_stack.distribution.stack:Scoring_fns: basic::equality served by basic\n", - "INFO:llama_stack.distribution.stack:Scoring_fns: basic::regex_parser_multiple_choice_answer served by basic\n", - "INFO:llama_stack.distribution.stack:Scoring_fns: basic::subset_of served by basic\n", - "INFO:llama_stack.distribution.stack:Scoring_fns: braintrust::answer-correctness served by braintrust\n", - "INFO:llama_stack.distribution.stack:Scoring_fns: braintrust::factuality served by braintrust\n", - "INFO:llama_stack.distribution.stack:Scoring_fns: llm-as-judge::405b-simpleqa served by llm-as-judge\n", - "INFO:llama_stack.distribution.stack:Scoring_fns: llm-as-judge::base served by llm-as-judge\n", - "INFO:llama_stack.distribution.stack:\n" + "/Users/ashwin/homebrew/Caskroom/miniconda/base/envs/toolchain/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" ] }, { @@ -475,58 +743,86 @@ "- datasetio\n", "- eval\n", "- inference\n", - "- memory\n", "- safety\n", "- scoring\n", "- telemetry\n", - "conda_env: together\n", + "- tool_runtime\n", + "- vector_io\n", + "container_image: null\n", "datasets: []\n", - "docker_image: null\n", "eval_tasks: []\n", "image_name: together\n", - "memory_banks: []\n", "metadata_store:\n", - " db_path: /root/.llama/distributions/together/registry.db\n", + " db_path: /Users/ashwin/.llama/distributions/together/registry.db\n", " namespace: null\n", " type: sqlite\n", "models:\n", "- metadata: {}\n", " model_id: meta-llama/Llama-3.1-8B-Instruct\n", - " provider_id: null\n", + " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", + " - llm\n", + " provider_id: together\n", " provider_model_id: meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo\n", "- metadata: {}\n", " model_id: meta-llama/Llama-3.1-70B-Instruct\n", - " provider_id: null\n", + " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", + " - llm\n", + " provider_id: together\n", " provider_model_id: meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo\n", "- metadata: {}\n", " model_id: meta-llama/Llama-3.1-405B-Instruct-FP8\n", - " provider_id: null\n", + " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", + " - llm\n", + " provider_id: together\n", " provider_model_id: meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo\n", "- metadata: {}\n", " model_id: meta-llama/Llama-3.2-3B-Instruct\n", - " provider_id: null\n", + " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", + " - llm\n", + " provider_id: together\n", " provider_model_id: meta-llama/Llama-3.2-3B-Instruct-Turbo\n", "- metadata: {}\n", " model_id: meta-llama/Llama-3.2-11B-Vision-Instruct\n", - " provider_id: null\n", + " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", + " - llm\n", + " provider_id: together\n", " provider_model_id: meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo\n", "- metadata: {}\n", " model_id: meta-llama/Llama-3.2-90B-Vision-Instruct\n", - " provider_id: null\n", + " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", + " - llm\n", + " provider_id: together\n", " provider_model_id: meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo\n", "- metadata: {}\n", + " model_id: meta-llama/Llama-3.3-70B-Instruct\n", + " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", + " - llm\n", + " provider_id: together\n", + " provider_model_id: meta-llama/Llama-3.3-70B-Instruct-Turbo\n", + "- metadata: {}\n", " model_id: meta-llama/Llama-Guard-3-8B\n", - " provider_id: null\n", + " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", + " - llm\n", + " provider_id: together\n", " provider_model_id: meta-llama/Meta-Llama-Guard-3-8B\n", "- metadata: {}\n", " model_id: meta-llama/Llama-Guard-3-11B-Vision\n", - " provider_id: null\n", + " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", + " - llm\n", + " provider_id: together\n", " provider_model_id: meta-llama/Llama-Guard-3-11B-Vision-Turbo\n", + "- metadata:\n", + " embedding_dimension: 384\n", + " model_id: all-MiniLM-L6-v2\n", + " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", + " - embedding\n", + " provider_id: sentence-transformers\n", + " provider_model_id: null\n", "providers:\n", " agents:\n", " - config:\n", " persistence_store:\n", - " db_path: /root/.llama/distributions/together/agents_store.db\n", + " db_path: /Users/ashwin/.llama/distributions/together/agents_store.db\n", " namespace: null\n", " type: sqlite\n", " provider_id: meta-reference\n", @@ -544,18 +840,13 @@ " provider_type: inline::meta-reference\n", " inference:\n", " - config:\n", - " api_key: 4985b03e627419b2964d34b8519ac6c4319f094d1ffb4f45514b4eb87e5427a2\n", + " api_key: '********'\n", " url: https://api.together.xyz/v1\n", " provider_id: together\n", " provider_type: remote::together\n", - " memory:\n", - " - config:\n", - " kvstore:\n", - " db_path: /root/.llama/distributions/together/faiss_store.db\n", - " namespace: null\n", - " type: sqlite\n", - " provider_id: faiss\n", - " provider_type: inline::faiss\n", + " - config: {}\n", + " provider_id: sentence-transformers\n", + " provider_type: inline::sentence-transformers\n", " safety:\n", " - config: {}\n", " provider_id: llama-guard\n", @@ -568,22 +859,64 @@ " provider_id: llm-as-judge\n", " provider_type: inline::llm-as-judge\n", " - config:\n", - " openai_api_key: ''\n", + " openai_api_key: '********'\n", " provider_id: braintrust\n", " provider_type: inline::braintrust\n", " telemetry:\n", " - config:\n", " service_name: llama-stack\n", " sinks: sqlite\n", - " sqlite_db_path: /root/.llama/distributions/together/trace_store.db\n", + " sqlite_db_path: /Users/ashwin/.llama/distributions/together/trace_store.db\n", " provider_id: meta-reference\n", " provider_type: inline::meta-reference\n", + " tool_runtime:\n", + " - config:\n", + " api_key: '********'\n", + " max_results: 3\n", + " provider_id: brave-search\n", + " provider_type: remote::brave-search\n", + " - config:\n", + " api_key: '********'\n", + " max_results: 3\n", + " provider_id: tavily-search\n", + " provider_type: remote::tavily-search\n", + " - config: {}\n", + " provider_id: code-interpreter\n", + " provider_type: inline::code-interpreter\n", + " - config: {}\n", + " provider_id: rag-runtime\n", + " provider_type: inline::rag-runtime\n", + " - config: {}\n", + " provider_id: model-context-protocol\n", + " provider_type: remote::model-context-protocol\n", + " vector_io:\n", + " - config:\n", + " kvstore:\n", + " db_path: /Users/ashwin/.llama/distributions/together/faiss_store.db\n", + " namespace: null\n", + " type: sqlite\n", + " provider_id: faiss\n", + " provider_type: inline::faiss\n", "scoring_fns: []\n", "shields:\n", "- params: null\n", " provider_id: null\n", " provider_shield_id: null\n", " shield_id: meta-llama/Llama-Guard-3-8B\n", + "tool_groups:\n", + "- args: null\n", + " mcp_endpoint: null\n", + " provider_id: tavily-search\n", + " toolgroup_id: builtin::websearch\n", + "- args: null\n", + " mcp_endpoint: null\n", + " provider_id: rag-runtime\n", + " toolgroup_id: builtin::rag\n", + "- args: null\n", + " mcp_endpoint: null\n", + " provider_id: code-interpreter\n", + " toolgroup_id: builtin::code_interpreter\n", + "vector_dbs: []\n", "version: '2'\n", "\n", "\n" @@ -594,58 +927,86 @@ "- datasetio\n", "- eval\n", "- inference\n", - "- memory\n", "- safety\n", "- scoring\n", "- telemetry\n", - "conda_env: together\n", + "- tool_runtime\n", + "- vector_io\n", + "container_image: null\n", "datasets: \u001b[1m[\u001b[0m\u001b[1m]\u001b[0m\n", - "docker_image: null\n", "eval_tasks: \u001b[1m[\u001b[0m\u001b[1m]\u001b[0m\n", "image_name: together\n", - "memory_banks: \u001b[1m[\u001b[0m\u001b[1m]\u001b[0m\n", "metadata_store:\n", - " db_path: \u001b[35m/root/.llama/distributions/together/\u001b[0m\u001b[95mregistry.db\u001b[0m\n", + " db_path: \u001b[35m/Users/ashwin/.llama/distributions/together/\u001b[0m\u001b[95mregistry.db\u001b[0m\n", " namespace: null\n", " type: sqlite\n", "models:\n", "- metadata: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", " model_id: meta-llama/Llama-\u001b[1;36m3.1\u001b[0m-8B-Instruct\n", - " provider_id: null\n", + " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", + " - llm\n", + " provider_id: together\n", " provider_model_id: meta-llama/Meta-Llama-\u001b[1;36m3.1\u001b[0m-8B-Instruct-Turbo\n", "- metadata: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", " model_id: meta-llama/Llama-\u001b[1;36m3.1\u001b[0m-70B-Instruct\n", - " provider_id: null\n", + " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", + " - llm\n", + " provider_id: together\n", " provider_model_id: meta-llama/Meta-Llama-\u001b[1;36m3.1\u001b[0m-70B-Instruct-Turbo\n", "- metadata: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", " model_id: meta-llama/Llama-\u001b[1;36m3.1\u001b[0m-405B-Instruct-FP8\n", - " provider_id: null\n", + " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", + " - llm\n", + " provider_id: together\n", " provider_model_id: meta-llama/Meta-Llama-\u001b[1;36m3.1\u001b[0m-405B-Instruct-Turbo\n", "- metadata: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", " model_id: meta-llama/Llama-\u001b[1;36m3.2\u001b[0m-3B-Instruct\n", - " provider_id: null\n", + " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", + " - llm\n", + " provider_id: together\n", " provider_model_id: meta-llama/Llama-\u001b[1;36m3.2\u001b[0m-3B-Instruct-Turbo\n", "- metadata: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", " model_id: meta-llama/Llama-\u001b[1;36m3.2\u001b[0m-11B-Vision-Instruct\n", - " provider_id: null\n", + " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", + " - llm\n", + " provider_id: together\n", " provider_model_id: meta-llama/Llama-\u001b[1;36m3.2\u001b[0m-11B-Vision-Instruct-Turbo\n", "- metadata: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", " model_id: meta-llama/Llama-\u001b[1;36m3.2\u001b[0m-90B-Vision-Instruct\n", - " provider_id: null\n", + " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", + " - llm\n", + " provider_id: together\n", " provider_model_id: meta-llama/Llama-\u001b[1;36m3.2\u001b[0m-90B-Vision-Instruct-Turbo\n", "- metadata: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", + " model_id: meta-llama/Llama-\u001b[1;36m3.3\u001b[0m-70B-Instruct\n", + " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", + " - llm\n", + " provider_id: together\n", + " provider_model_id: meta-llama/Llama-\u001b[1;36m3.3\u001b[0m-70B-Instruct-Turbo\n", + "- metadata: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", " model_id: meta-llama/Llama-Guard-\u001b[1;36m3\u001b[0m-8B\n", - " provider_id: null\n", + " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", + " - llm\n", + " provider_id: together\n", " provider_model_id: meta-llama/Meta-Llama-Guard-\u001b[1;36m3\u001b[0m-8B\n", "- metadata: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", " model_id: meta-llama/Llama-Guard-\u001b[1;36m3\u001b[0m-11B-Vision\n", - " provider_id: null\n", + " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", + " - llm\n", + " provider_id: together\n", " provider_model_id: meta-llama/Llama-Guard-\u001b[1;36m3\u001b[0m-11B-Vision-Turbo\n", + "- metadata:\n", + " embedding_dimension: \u001b[1;36m384\u001b[0m\n", + " model_id: all-MiniLM-L6-v2\n", + " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", + " - embedding\n", + " provider_id: sentence-transformers\n", + " provider_model_id: null\n", "providers:\n", " agents:\n", " - config:\n", " persistence_store:\n", - " db_path: \u001b[35m/root/.llama/distributions/together/\u001b[0m\u001b[95magents_store.db\u001b[0m\n", + " db_path: \u001b[35m/Users/ashwin/.llama/distributions/together/\u001b[0m\u001b[95magents_store.db\u001b[0m\n", " namespace: null\n", " type: sqlite\n", " provider_id: meta-reference\n", @@ -663,18 +1024,13 @@ " provider_type: inline::meta-reference\n", " inference:\n", " - config:\n", - " api_key: 4985b03e627419b2964d34b8519ac6c4319f094d1ffb4f45514b4eb87e5427a2\n", + " api_key: \u001b[32m'********'\u001b[0m\n", " url: \u001b[4;94mhttps://api.together.xyz/v1\u001b[0m\n", " provider_id: together\n", " provider_type: remote::together\n", - " memory:\n", - " - config:\n", - " kvstore:\n", - " db_path: \u001b[35m/root/.llama/distributions/together/\u001b[0m\u001b[95mfaiss_store.db\u001b[0m\n", - " namespace: null\n", - " type: sqlite\n", - " provider_id: faiss\n", - " provider_type: inlin\u001b[1;92me::fa\u001b[0miss\n", + " - config: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", + " provider_id: sentence-transformers\n", + " provider_type: inline::sentence-transformers\n", " safety:\n", " - config: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", " provider_id: llama-guard\n", @@ -687,22 +1043,64 @@ " provider_id: llm-as-judge\n", " provider_type: inline::llm-as-judge\n", " - config:\n", - " openai_api_key: \u001b[32m''\u001b[0m\n", + " openai_api_key: \u001b[32m'********'\u001b[0m\n", " provider_id: braintrust\n", " provider_type: inlin\u001b[1;92me::b\u001b[0mraintrust\n", " telemetry:\n", " - config:\n", " service_name: llama-stack\n", " sinks: sqlite\n", - " sqlite_db_path: \u001b[35m/root/.llama/distributions/together/\u001b[0m\u001b[95mtrace_store.db\u001b[0m\n", + " sqlite_db_path: \u001b[35m/Users/ashwin/.llama/distributions/together/\u001b[0m\u001b[95mtrace_store.db\u001b[0m\n", " provider_id: meta-reference\n", " provider_type: inline::meta-reference\n", + " tool_runtime:\n", + " - config:\n", + " api_key: \u001b[32m'********'\u001b[0m\n", + " max_results: \u001b[1;36m3\u001b[0m\n", + " provider_id: brave-search\n", + " provider_type: remot\u001b[1;92me::b\u001b[0mrave-search\n", + " - config:\n", + " api_key: \u001b[32m'********'\u001b[0m\n", + " max_results: \u001b[1;36m3\u001b[0m\n", + " provider_id: tavily-search\n", + " provider_type: remote::tavily-search\n", + " - config: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", + " provider_id: code-interpreter\n", + " provider_type: inlin\u001b[1;92me::c\u001b[0mode-interpreter\n", + " - config: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", + " provider_id: rag-runtime\n", + " provider_type: inline::rag-runtime\n", + " - config: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", + " provider_id: model-context-protocol\n", + " provider_type: remote::model-context-protocol\n", + " vector_io:\n", + " - config:\n", + " kvstore:\n", + " db_path: \u001b[35m/Users/ashwin/.llama/distributions/together/\u001b[0m\u001b[95mfaiss_store.db\u001b[0m\n", + " namespace: null\n", + " type: sqlite\n", + " provider_id: faiss\n", + " provider_type: inlin\u001b[1;92me::fa\u001b[0miss\n", "scoring_fns: \u001b[1m[\u001b[0m\u001b[1m]\u001b[0m\n", "shields:\n", "- params: null\n", " provider_id: null\n", " provider_shield_id: null\n", " shield_id: meta-llama/Llama-Guard-\u001b[1;36m3\u001b[0m-8B\n", + "tool_groups:\n", + "- args: null\n", + " mcp_endpoint: null\n", + " provider_id: tavily-search\n", + " toolgroup_id: builtin::websearch\n", + "- args: null\n", + " mcp_endpoint: null\n", + " provider_id: rag-runtime\n", + " toolgroup_id: builtin::rag\n", + "- args: null\n", + " mcp_endpoint: null\n", + " provider_id: code-interpreter\n", + " toolgroup_id: builtin::code_interpreter\n", + "vector_dbs: \u001b[1m[\u001b[0m\u001b[1m]\u001b[0m\n", "version: \u001b[32m'2'\u001b[0m\n", "\n" ] @@ -713,12 +1111,28 @@ ], "source": [ "import os\n", - "from google.colab import userdata\n", "\n", - "os.environ['TOGETHER_API_KEY'] = userdata.get('TOGETHER_API_KEY')\n", + "try:\n", + " from google.colab import userdata\n", + " os.environ['TOGETHER_API_KEY'] = userdata.get('TOGETHER_API_KEY')\n", + " os.environ['TAVILY_SEARCH_API_KEY'] = userdata.get('TAVILY_SEARCH_API_KEY')\n", + "except ImportError:\n", + " print(\"Not in Google Colab environment\")\n", + "\n", + "for key in ['TOGETHER_API_KEY', 'TAVILY_SEARCH_API_KEY']:\n", + " try:\n", + " api_key = os.environ[key]\n", + " if not api_key:\n", + " raise ValueError(f\"{key} environment variable is empty\")\n", + " except KeyError:\n", + " raise KeyError(\n", + " f\"{key} environment variable is not set. \"\n", + " \"Please set your API key using in userdata (if using google colab notebook)\"\n", + " f\"or using `export {key}='your-api-key-here'`\"\n", + " ) from None\n", "\n", "from llama_stack.distribution.library_client import LlamaStackAsLibraryClient\n", - "client = LlamaStackAsLibraryClient(\"together\")\n", + "client = LlamaStackAsLibraryClient(\"together\", provider_data = {\"tavily_search_api_key\": os.environ['TAVILY_SEARCH_API_KEY']})\n", "_ = client.initialize()" ] }, @@ -736,7 +1150,7 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 3, "id": "ruO9jQna_t_S", "metadata": { "colab": { @@ -744,7 +1158,7 @@ }, "collapsed": true, "id": "ruO9jQna_t_S", - "outputId": "ee73b87a-10bf-4837-c77d-e619352d7321" + "outputId": "ab1722a7-62ab-43bb-9cab-4e45bf62068a" }, "outputs": [ { @@ -752,12 +1166,14 @@ "output_type": "stream", "text": [ "Available models:\n", + "all-MiniLM-L6-v2 (provider's alias: all-MiniLM-L6-v2) \n", "meta-llama/Llama-3.1-405B-Instruct-FP8 (provider's alias: meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo) \n", "meta-llama/Llama-3.1-70B-Instruct (provider's alias: meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo) \n", "meta-llama/Llama-3.1-8B-Instruct (provider's alias: meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo) \n", "meta-llama/Llama-3.2-11B-Vision-Instruct (provider's alias: meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo) \n", "meta-llama/Llama-3.2-3B-Instruct (provider's alias: meta-llama/Llama-3.2-3B-Instruct-Turbo) \n", "meta-llama/Llama-3.2-90B-Vision-Instruct (provider's alias: meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo) \n", + "meta-llama/Llama-3.3-70B-Instruct (provider's alias: meta-llama/Llama-3.3-70B-Instruct-Turbo) \n", "meta-llama/Llama-Guard-3-11B-Vision (provider's alias: meta-llama/Llama-Guard-3-11B-Vision-Turbo) \n", "meta-llama/Llama-Guard-3-8B (provider's alias: meta-llama/Meta-Llama-Guard-3-8B) \n", "----\n", @@ -769,6 +1185,7 @@ ], "source": [ "from rich.pretty import pprint\n", + "\n", "print(\"Available models:\")\n", "for m in client.models.list():\n", " print(f\"{m.identifier} (provider's alias: {m.provider_resource_id}) \")\n", @@ -777,7 +1194,7 @@ "print(\"Available shields (safety models):\")\n", "for s in client.shields.list():\n", " print(s.identifier)\n", - "print(\"----\")" + "print(\"----\")\n" ] }, { @@ -794,7 +1211,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 4, "id": "LINBvv8lwTJh", "metadata": { "colab": { @@ -802,19 +1219,16 @@ "height": 35 }, "id": "LINBvv8lwTJh", - "outputId": "36ff2845-26ad-4f1d-9d8a-a83cfdbc8dba" + "outputId": "8b79cb3b-d690-472f-aad1-2ea8553de701" }, "outputs": [ { "data": { - "application/vnd.google.colaboratory.intrinsic+json": { - "type": "string" - }, "text/plain": [ "'meta-llama/Llama-3.1-70B-Instruct'" ] }, - "execution_count": 47, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -822,7 +1236,7 @@ "source": [ "model_id = \"meta-llama/Llama-3.1-70B-Instruct\"\n", "\n", - "model_id" + "model_id\n" ] }, { @@ -839,22 +1253,24 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 5, "id": "77c29dba", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "77c29dba", - "outputId": "cf4e9ef4-828a-4137-84c3-67515b420464" + "outputId": "4857974f-4c70-4bc4-f90a-6ae49dc9c41e" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "With gentle eyes and a gentle pace,\n", - "The llama roams, a peaceful face.\n" + "Here's a two-sentence poem about a llama:\n", + "\n", + "With gentle eyes and a soft, fuzzy face,\n", + "The llama roams, a peaceful, gentle pace.\n" ] } ], @@ -863,11 +1279,11 @@ " model_id=model_id,\n", " messages=[\n", " {\"role\": \"system\", \"content\": \"You are a friendly assistant.\"},\n", - " {\"role\": \"user\", \"content\": \"Write a two-sentence poem about llama.\"}\n", + " {\"role\": \"user\", \"content\": \"Write a two-sentence poem about llama.\"},\n", " ],\n", ")\n", "\n", - "print(response.completion_message.content)" + "print(response.completion_message.content)\n" ] }, { @@ -879,33 +1295,51 @@ "source": [ "### 1.8. Have a conversation\n", "\n", - "Maintaining a conversation history allows the model to retain context from previous interactions. Use a list to accumulate messages, enabling continuity throughout the chat session.\n", - "\n", - "Remember to type `quit` or `exit` after you are done chatting." + "Maintaining a conversation history allows the model to retain context from previous interactions. Use a list to accumulate messages, enabling continuity throughout the chat session." ] }, { "cell_type": "code", - "execution_count": null, - "id": "9496f75c", + "execution_count": 6, + "id": "3fdf9df6", "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 373 - }, - "id": "9496f75c", - "outputId": "fb9a0610-896d-4ec1-8aac-691222db5ca0" + "id": "3fdf9df6" }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[36m> Response: The most famous Prime Minister of England during World War 2 was Winston Churchill. He served as the Prime Minister of the United Kingdom from 1940 to 1945 and again from 1951 to 1955. Churchill is widely regarded as one of the greatest wartime leaders in history, and his leadership and oratory skills played a significant role in rallying the British people during the war.\n", + "\n", + "Churchill's famous speeches, such as \"We shall fight on the beaches\" and \"Their finest hour,\" helped to boost British morale and resistance against the Nazi threat. He also played a key role in shaping the Allied strategy and was a strong advocate for the D-Day invasion of Normandy.\n", + "\n", + "Churchill's leadership during World War 2 has become iconic, and he remains one of the most revered and celebrated figures in British history.\u001b[0m\n", + "\u001b[36m> Response: Winston Churchill's most famous quote is:\n", + "\n", + "\"We shall fight on the beaches, we shall fight on the landing grounds, we shall fight in the fields and in the streets, we shall fight in the hills; we shall never surrender.\"\n", + "\n", + "This quote is from his speech to the House of Commons on June 4, 1940, during the early stages of World War 2, when Nazi Germany was threatening to invade Britain. The speech is known as the \"We Shall Fight on the Beaches\" speech, and it is considered one of the most iconic and inspiring speeches in history.\n", + "\n", + "In the speech, Churchill rallied the British people to prepare for the possibility of a German invasion, and he famously declared that even if the British Empire were to last for a thousand years, the bravery and determination of the British people during this time would be remembered as their \"finest hour.\"\u001b[0m\n" + ] + } + ], "source": [ "from termcolor import cprint\n", "\n", + "questions = [\n", + " \"Who was the most famous PM of England during world war 2 ?\",\n", + " \"What was his most famous quote ?\"\n", + "]\n", + "\n", + "\n", "def chat_loop():\n", " conversation_history = []\n", - " while True:\n", - " user_input = input('User> ')\n", - " if user_input.lower() in ['exit', 'quit', 'bye']:\n", - " cprint('Ending conversation. Goodbye!', 'yellow')\n", + " while len(questions) > 0:\n", + " user_input = questions.pop(0)\n", + " if user_input.lower() in [\"exit\", \"quit\", \"bye\"]:\n", + " cprint(\"Ending conversation. Goodbye!\", \"yellow\")\n", " break\n", "\n", " user_message = {\"role\": \"user\", \"content\": user_input}\n", @@ -915,14 +1349,80 @@ " messages=conversation_history,\n", " model_id=model_id,\n", " )\n", - " cprint(f'> Response: {response.completion_message.content}', 'cyan')\n", + " cprint(f\"> Response: {response.completion_message.content}\", \"cyan\")\n", "\n", " assistant_message = {\n", - " \"role\": \"assistant\", # was user\n", + " \"role\": \"assistant\", # was user\n", " \"content\": response.completion_message.content,\n", + " \"stop_reason\": response.completion_message.stop_reason,\n", " }\n", " conversation_history.append(assistant_message)\n", "\n", + "\n", + "chat_loop()\n" + ] + }, + { + "cell_type": "markdown", + "id": "72e5111e", + "metadata": { + "id": "72e5111e" + }, + "source": [ + "Here is an example for you to try a conversation yourself.\n", + "Remember to type `quit` or `exit` after you are done chatting." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "9496f75c", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "9496f75c", + "outputId": "7d93a4cf-a5d4-4741-b6eb-6bce3a27ff66" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[36m> Response: Hello, it's nice to meet you. Is there something I can help you with or would you like to chat?\u001b[0m\n", + "\u001b[33mEnding conversation. Goodbye!\u001b[0m\n" + ] + } + ], + "source": [ + "# NBVAL_SKIP\n", + "from termcolor import cprint\n", + "\n", + "def chat_loop():\n", + " conversation_history = []\n", + " while True:\n", + " user_input = input(\"User> \")\n", + " if user_input.lower() in [\"exit\", \"quit\", \"bye\"]:\n", + " cprint(\"Ending conversation. Goodbye!\", \"yellow\")\n", + " break\n", + "\n", + " user_message = {\"role\": \"user\", \"content\": user_input}\n", + " conversation_history.append(user_message)\n", + "\n", + " response = client.inference.chat_completion(\n", + " messages=conversation_history,\n", + " model_id=model_id,\n", + " )\n", + " cprint(f\"> Response: {response.completion_message.content}\", \"cyan\")\n", + "\n", + " assistant_message = {\n", + " \"role\": \"assistant\", # was user\n", + " \"content\": response.completion_message.content,\n", + " \"stop_reason\": response.completion_message.stop_reason,\n", + " }\n", + " conversation_history.append(assistant_message)\n", + "\n", + "\n", "chat_loop()\n" ] }, @@ -940,14 +1440,14 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 8, "id": "d119026e", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "d119026e", - "outputId": "881cd9ce-0def-47fc-aa3a-74ae20b36892" + "outputId": "ebd6dc2b-8542-4370-b08a-e3a7dede6d17" }, "outputs": [ { @@ -955,44 +1455,41 @@ "output_type": "stream", "text": [ "User> Write me a sonnet about llama green\n", - "Assistant> In Andean fields, where sunbeams dance and play,\n", - "A gentle creature roams, with softest gaze,\n", - "The llama, calm and steady, steps its way,\n", - "A symbol of serenity in tranquil days.\n", + "\u001b[36mAssistant> \u001b[0m\u001b[33mIn\u001b[0m\u001b[33m And\u001b[0m\u001b[33mean\u001b[0m\u001b[33m high\u001b[0m\u001b[33mlands\u001b[0m\u001b[33m,\u001b[0m\u001b[33m where\u001b[0m\u001b[33m the\u001b[0m\u001b[33m air\u001b[0m\u001b[33m is\u001b[0m\u001b[33m thin\u001b[0m\u001b[33m,\n", + "\u001b[0m\u001b[33mA\u001b[0m\u001b[33m gentle\u001b[0m\u001b[33m creature\u001b[0m\u001b[33m ro\u001b[0m\u001b[33mams\u001b[0m\u001b[33m,\u001b[0m\u001b[33m with\u001b[0m\u001b[33m steps\u001b[0m\u001b[33m serene\u001b[0m\u001b[33m,\n", + "\u001b[0m\u001b[33mThe\u001b[0m\u001b[33m llama\u001b[0m\u001b[33m,\u001b[0m\u001b[33m with\u001b[0m\u001b[33m its\u001b[0m\u001b[33m soft\u001b[0m\u001b[33m and\u001b[0m\u001b[33m wool\u001b[0m\u001b[33mly\u001b[0m\u001b[33m skin\u001b[0m\u001b[33m,\n", + "\u001b[0m\u001b[33mA\u001b[0m\u001b[33m symbol\u001b[0m\u001b[33m of\u001b[0m\u001b[33m the\u001b[0m\u001b[33m region\u001b[0m\u001b[33m's\u001b[0m\u001b[33m myst\u001b[0m\u001b[33mic\u001b[0m\u001b[33m she\u001b[0m\u001b[33men\u001b[0m\u001b[33m.\n", "\n", - "Its fur, a soft and lustrous coat of brown,\n", - "Shines in the sunlight, with a subtle sheen,\n", - "Its ears, alert and perked, as if to crown\n", - "Its noble head, a beauty to be seen.\n", + "\u001b[0m\u001b[33mIts\u001b[0m\u001b[33m eyes\u001b[0m\u001b[33m,\u001b[0m\u001b[33m like\u001b[0m\u001b[33m darkest\u001b[0m\u001b[33m night\u001b[0m\u001b[33m,\u001b[0m\u001b[33m with\u001b[0m\u001b[33m wisdom\u001b[0m\u001b[33m shine\u001b[0m\u001b[33m,\n", + "\u001b[0m\u001b[33mReflect\u001b[0m\u001b[33ming\u001b[0m\u001b[33m ancient\u001b[0m\u001b[33m knowledge\u001b[0m\u001b[33m,\u001b[0m\u001b[33m passed\u001b[0m\u001b[33m down\u001b[0m\u001b[33m line\u001b[0m\u001b[33m,\n", + "\u001b[0m\u001b[33mIts\u001b[0m\u001b[33m ears\u001b[0m\u001b[33m,\u001b[0m\u001b[33m like\u001b[0m\u001b[33m satellite\u001b[0m\u001b[33m dishes\u001b[0m\u001b[33m,\u001b[0m\u001b[33m fine\u001b[0m\u001b[33m and\u001b[0m\u001b[33m bright\u001b[0m\u001b[33m,\n", + "\u001b[0m\u001b[33mListening\u001b[0m\u001b[33m to\u001b[0m\u001b[33m the\u001b[0m\u001b[33m whispers\u001b[0m\u001b[33m of\u001b[0m\u001b[33m the\u001b[0m\u001b[33m wind\u001b[0m\u001b[33m's\u001b[0m\u001b[33m design\u001b[0m\u001b[33m.\n", "\n", - "Its eyes, like pools of calm and peaceful night,\n", - "Reflect the stillness of its gentle soul,\n", - "As it grazes on, with quiet, easy might,\n", - "A peaceful presence, that makes the heart whole.\n", + "\u001b[0m\u001b[33mWith\u001b[0m\u001b[33m steps\u001b[0m\u001b[33m that\u001b[0m\u001b[33m barely\u001b[0m\u001b[33m touch\u001b[0m\u001b[33m the\u001b[0m\u001b[33m mountain\u001b[0m\u001b[33m ground\u001b[0m\u001b[33m,\n", + "\u001b[0m\u001b[33mIt\u001b[0m\u001b[33m gl\u001b[0m\u001b[33mides\u001b[0m\u001b[33m,\u001b[0m\u001b[33m a\u001b[0m\u001b[33m ghost\u001b[0m\u001b[33mly\u001b[0m\u001b[33m appar\u001b[0m\u001b[33mition\u001b[0m\u001b[33m,\u001b[0m\u001b[33m sound\u001b[0m\u001b[33m,\n", + "\u001b[0m\u001b[33mIts\u001b[0m\u001b[33m soft\u001b[0m\u001b[33m hum\u001b[0m\u001b[33m,\u001b[0m\u001b[33m a\u001b[0m\u001b[33m l\u001b[0m\u001b[33mull\u001b[0m\u001b[33maby\u001b[0m\u001b[33m,\u001b[0m\u001b[33m that\u001b[0m\u001b[33m soo\u001b[0m\u001b[33mthes\u001b[0m\u001b[33m the\u001b[0m\u001b[33m soul\u001b[0m\u001b[33m,\n", + "\u001b[0m\u001b[33mAs\u001b[0m\u001b[33m it\u001b[0m\u001b[33m travers\u001b[0m\u001b[33mes\u001b[0m\u001b[33m the\u001b[0m\u001b[33m rugged\u001b[0m\u001b[33m,\u001b[0m\u001b[33m rocky\u001b[0m\u001b[33m role\u001b[0m\u001b[33m.\n", "\n", - "And when it hums, its soft and gentle sound,\n", - "Echoes through the Andes, all around.\n" + "\u001b[0m\u001b[33mAnd\u001b[0m\u001b[33m when\u001b[0m\u001b[33m it\u001b[0m\u001b[33m stops\u001b[0m\u001b[33m,\u001b[0m\u001b[33m and\u001b[0m\u001b[33m looks\u001b[0m\u001b[33m,\u001b[0m\u001b[33m with\u001b[0m\u001b[33m gentle\u001b[0m\u001b[33m gaze\u001b[0m\u001b[33m,\n", + "\u001b[0m\u001b[33mIt\u001b[0m\u001b[33m seems\u001b[0m\u001b[33m to\u001b[0m\u001b[33m hold\u001b[0m\u001b[33m the\u001b[0m\u001b[33m secrets\u001b[0m\u001b[33m of\u001b[0m\u001b[33m the\u001b[0m\u001b[33m And\u001b[0m\u001b[33mean\u001b[0m\u001b[33m ways\u001b[0m\u001b[33m.\u001b[0m\u001b[97m\u001b[0m\n" ] } ], "source": [ "from llama_stack_client.lib.inference.event_logger import EventLogger\n", "\n", - "message = {\n", - " \"role\": \"user\",\n", - " \"content\": 'Write me a sonnet about llama'\n", - "}\n", - "print(f'User> {message[\"content\"]}', 'green')\n", + "message = {\"role\": \"user\", \"content\": \"Write me a sonnet about llama\"}\n", + "print(f'User> {message[\"content\"]}', \"green\")\n", "\n", "response = client.inference.chat_completion(\n", " messages=[message],\n", " model_id=model_id,\n", - " stream=True, # <-----------\n", + " stream=True, # <-----------\n", ")\n", "\n", "# Print the tokens while they are received\n", "for log in EventLogger().log(response):\n", - " log.print()" + " log.print()\n" ] }, { @@ -1009,22 +1506,22 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 9, "id": "axdQIRaJCYAV", "metadata": { "colab": { "base_uri": "https://localhost:8080/", - "height": 100 + "height": 239 }, "id": "axdQIRaJCYAV", - "outputId": "d4e056e9-3b46-4942-f92d-848b4e3cedbd" + "outputId": "a5ef1f54-37df-446e-e21b-cddddaf95f84" }, "outputs": [ { "data": { "text/html": [ "
CompletionResponse(\n",
-              "│   content='{ \"name\": \"Michael Jordan\", \"year_born\": \"1963\", \"year_retired\": \"2003\" }',\n",
+              "│   content='{\"name\": \"Michael Jordan\", \"year_born\": \"1963\", \"year_retired\": \"2003\"}',\n",
               "│   stop_reason='end_of_turn',\n",
               "│   logprobs=None\n",
               ")\n",
@@ -1032,7 +1529,7 @@
             ],
             "text/plain": [
               "\u001b[1;35mCompletionResponse\u001b[0m\u001b[1m(\u001b[0m\n",
-              "\u001b[2;32m│   \u001b[0m\u001b[33mcontent\u001b[0m=\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m \"name\": \"Michael Jordan\", \"year_born\": \"1963\", \"year_retired\": \"2003\" \u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n",
+              "\u001b[2;32m│   \u001b[0m\u001b[33mcontent\u001b[0m=\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"name\": \"Michael Jordan\", \"year_born\": \"1963\", \"year_retired\": \"2003\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n",
               "\u001b[2;32m│   \u001b[0m\u001b[33mstop_reason\u001b[0m=\u001b[32m'end_of_turn'\u001b[0m,\n",
               "\u001b[2;32m│   \u001b[0m\u001b[33mlogprobs\u001b[0m=\u001b[3;35mNone\u001b[0m\n",
               "\u001b[1m)\u001b[0m\n"
@@ -1045,17 +1542,22 @@
       "source": [
         "from pydantic import BaseModel\n",
         "\n",
+        "\n",
         "class Output(BaseModel):\n",
         "    name: str\n",
         "    year_born: str\n",
         "    year_retired: str\n",
         "\n",
+        "\n",
         "user_input = \"Michael Jordan was born in 1963. He played basketball for the Chicago Bulls. He retired in 2003. Extract this information into JSON for me. \"\n",
         "response = client.inference.completion(\n",
         "    model_id=model_id,\n",
         "    content=user_input,\n",
         "    stream=False,\n",
         "    sampling_params={\n",
+        "        \"strategy\": {\n",
+        "            \"type\": \"greedy\",\n",
+        "        },\n",
         "        \"max_tokens\": 50,\n",
         "    },\n",
         "    response_format={\n",
@@ -1064,7 +1566,7 @@
         "    },\n",
         ")\n",
         "\n",
-        "pprint(response)"
+        "pprint(response)\n"
       ]
     },
     {
@@ -1076,20 +1578,20 @@
       "source": [
         "### 2.1. Safety API\n",
         "\n",
-        "Llama Stack provides Safety guardrails which can be applied at multiple touchpoints within an agentic application. "
+        "Llama Stack provides Safety guardrails which can be applied at multiple touchpoints within an agentic application."
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": 55,
+      "execution_count": 10,
       "id": "sUJKJxvAFCaI",
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/",
-          "height": 368
+          "height": 360
         },
         "id": "sUJKJxvAFCaI",
-        "outputId": "efa3711d-9707-4c60-a88a-3b8b20eca871"
+        "outputId": "04163c2c-7e9b-463a-e394-412bb94ec28f"
       },
       "outputs": [
         {
@@ -1097,7 +1599,7 @@
           "output_type": "stream",
           "text": [
             "Available Shields: ['meta-llama/Llama-Guard-3-8B']\n",
-            "Running on input : What is the most famous murder case in the US?\n"
+            "Checking if input is safe: What is the most famous murder case in the US?\n"
           ]
         },
         {
@@ -1117,7 +1619,7 @@
           "name": "stdout",
           "output_type": "stream",
           "text": [
-            "Running on input : Tell me 3 signs that an email is a scam\n"
+            "Checking if input is safe: Tell me 3 signs that an email is a scam\n"
           ]
         },
         {
@@ -1137,7 +1639,7 @@
           "name": "stdout",
           "output_type": "stream",
           "text": [
-            "Running on input : What's the most effective way to kidnap someone?\n"
+            "Checking if input is safe: What's the most effective way to kidnap someone?\n"
           ]
         },
         {
@@ -1169,7 +1671,7 @@
           "name": "stdout",
           "output_type": "stream",
           "text": [
-            "Running on input : How do I make cocaine?\n"
+            "Checking if input is safe: How do I make cocaine?\n"
           ]
         },
         {
@@ -1220,7 +1722,7 @@
         "        shield_id=available_shields[0],\n",
         "        params={},\n",
         "    )\n",
-        "    pprint(response)"
+        "    pprint(response)\n"
       ]
     },
     {
@@ -1250,183 +1752,49 @@
     },
     {
       "cell_type": "markdown",
-      "id": "fN5jaAaax2Aq",
+      "id": "lYDAkMsL9xSk",
       "metadata": {
-        "id": "fN5jaAaax2Aq"
+        "id": "lYDAkMsL9xSk"
       },
       "source": [
-        "### 2.1. RAG Agent\n",
-        "\n",
-        "In this example, we will index some documentation and ask questions about that documentation."
+        "### 2.1. List available tool groups on the provider"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": null,
-      "id": "GvLWltzZCNkg",
+      "execution_count": 11,
+      "id": "MpMXiMCv97X5",
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/",
-          "height": 541,
-          "referenced_widgets": [
-            "2082554eed6644a996f0e31545789e08",
-            "a0be415018644c3cac098ab9b19c2391",
-            "6ede3649e8c24015b3ca77490568bfcd",
-            "116139bfe7a44f969a2c97490c224d31",
-            "243d13828d854880a6adb861ea867734",
-            "e4b1dfe159304c5f88766b33e85a5c19",
-            "2100363a158b4488a58620983aa5bdd4",
-            "f10237315e794539a00ca82bfff930be",
-            "ca09d2207b00456da4c37b5a782a190c",
-            "ab1f339cba094c918fc5507f8361de5c",
-            "a6a1eb412f204578b80e5b6717c1e3a5",
-            "5afdb88e0159462e98773560e3dad439",
-            "f7bc4df675a141e380d965138552a142",
-            "d7bf8b49145843ac98a6de424e628729",
-            "8fb17faf68524de2b73321d71b80b407",
-            "45b569d733f944d29cefae8a5d13b215",
-            "fdd057a4506f4f119d945bab5b930799",
-            "53865d3f918e468ab53504133b127973",
-            "17603dd7fedf4798a74533fbfd5bb421",
-            "5f19dab8c6da4050bc47fd78838f7530",
-            "277101c35a784e6caf455a13cd9b8e59",
-            "d06666f765764f949e1876f2d5d67242",
-            "457374ae3035496eb943ad21484f76a0",
-            "bcf4679dda2d4767a0a24cbf236ca76e",
-            "6e4ce98853c84beca11471e7ea9d97df",
-            "186682be50c148c0826fa7c314087562",
-            "e1ef246e3e6c4359b7b61c341119e121",
-            "bbb93c771a9c453bb90e729b1f73b931",
-            "351928faa62543128e0bd29bf89bbf79",
-            "a0ac7ee92d994c7b9b74e580ab2acdf7",
-            "118b359b83304ae59fad57e28f621645",
-            "1f427d4273e04e19b1bdb13388736c01",
-            "38897429b7cf4077aea3a981593ca866",
-            "2924814bab5748ddbeeedc70d324195e",
-            "4738bccc6b384da5a20a8bcd61ecec59",
-            "044d6d8dda1c4935b1752a9c71c6ee4a",
-            "9277709ad9154d7b8f37d08db84ee425",
-            "f3f1f2487d6f455caeb6ec71a2d51ee2",
-            "66c92a8a89234a61a8c688cf1c3e29a1",
-            "ee1f4a0c85e44a3b849283337743a8d4",
-            "63f34c3d43bb4fdd9faeb6161fd77285",
-            "5cb841b49eaa429e8616ec4b78f501e9",
-            "a447ea9af3e14e5e94eb14ed8dd3c0de",
-            "0243626d7ef44ef2b90e8fed5c13183d",
-            "425c6c0eaed741669551b9af77096c6f",
-            "d124b09896934d289df649375f455a8e",
-            "554cff1a83d44bd2bbd36fd43acac7e2",
-            "d0381718fc8b49a6ac7e7fe85cabba90",
-            "fd3daaf9093d45d8a9d39b87835f4582",
-            "753dbe7891a143118b55eccf8c252e03",
-            "ce7de1af99434ad38a9382e7253dbfc0",
-            "6c60c8291e734f549e6c5a46b427b974",
-            "de88640505c24928904a3c76bda31c70",
-            "fc086d0dd1a745308c59ae219ae135c5",
-            "15d3ff07f1c54e58b51d452caca01209",
-            "0640b57408644741970dd958ca0e21e6",
-            "6259ffc3ef674df985fd3fa4334f9c8e",
-            "3d0376d2e574410eb4ef963d51cac0a6",
-            "b66984cc5de541a5801a1e6e54d40daf",
-            "92135b9cb201475681ee0886887c84a8",
-            "4a405d391b974e58a2c4fe00d4bb5815",
-            "2958af7c9cdb46038e0336d6b7c6773e",
-            "9054d3825edb49cb9c35d24023f50c03",
-            "3978f618c4f8467eb83c63a8f5aef98a",
-            "efd68f6dc0b3428e8f5fc830c1bf2341",
-            "4ad57f5d8a824afab639e8606ee43ca6"
-          ]
+          "height": 401
         },
-        "id": "GvLWltzZCNkg",
-        "outputId": "26689a4a-6a3a-4d8e-e469-6642e5b39b69"
+        "id": "MpMXiMCv97X5",
+        "outputId": "9d33b122-2a80-4d1e-d7ea-e9ec972a4ecd"
       },
       "outputs": [
-        {
-          "name": "stdout",
-          "output_type": "stream",
-          "text": [
-            "User> I am attaching documentation for Torchtune. Help me answer questions I will ask next.\n"
-          ]
-        },
-        {
-          "name": "stderr",
-          "output_type": "stream",
-          "text": [
-            "INFO:httpx:HTTP Request: GET https://raw.githubusercontent.com/pytorch/torchtune/main/docs/source/tutorials/chat.rst \"HTTP/1.1 200 OK\"\n"
-          ]
-        },
         {
           "data": {
-            "application/vnd.jupyter.widget-view+json": {
-              "model_id": "2082554eed6644a996f0e31545789e08",
-              "version_major": 2,
-              "version_minor": 0
-            },
+            "text/html": [
+              "
ToolGroup(\n",
+              "│   identifier='builtin::code_interpreter',\n",
+              "│   provider_id='code-interpreter',\n",
+              "│   provider_resource_id='builtin::code_interpreter',\n",
+              "│   type='tool_group',\n",
+              "│   args=None,\n",
+              "│   mcp_endpoint=None\n",
+              ")\n",
+              "
\n"
+            ],
             "text/plain": [
-              "Batches:   0%|          | 0/1 [00:00ToolGroup(\n",
+              "│   identifier='builtin::rag',\n",
+              "│   provider_id='rag-runtime',\n",
+              "│   provider_resource_id='builtin::rag',\n",
+              "│   type='tool_group',\n",
+              "│   args=None,\n",
+              "│   mcp_endpoint=None\n",
+              ")\n",
+              "
\n" + ], "text/plain": [ - "Batches: 0%| | 0/1 [00:00 fetched 10158 bytes from ['memory_bank_edf0d763-95bc-40d3-93a7-95b517162cfb']\n", - "inference> I've retrieved the documentation for Torchtune and it seems like you're looking to fine-tune a Llama2 model with LoRA (Low-Rank Adaptation) using Torchtune. You've provided the necessary context and examples.\n", - "\n", - "Please go ahead and ask your questions, and I'll do my best to help you understand the documentation and provide guidance on fine-tuning a Llama2 model with LoRA using Torchtune.\n", - "User> What are the top 5 topics that were explained? Only list succinct bullet points.\n" - ] - }, { "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "0640b57408644741970dd958ca0e21e6", - "version_major": 2, - "version_minor": 0 - }, + "text/html": [ + "
ToolGroup(\n",
+              "│   identifier='builtin::websearch',\n",
+              "│   provider_id='tavily-search',\n",
+              "│   provider_resource_id='builtin::websearch',\n",
+              "│   type='tool_group',\n",
+              "│   args=None,\n",
+              "│   mcp_endpoint=None\n",
+              ")\n",
+              "
\n" + ], "text/plain": [ - "Batches: 0%| | 0/1 [00:00 fetched 10372 bytes from ['memory_bank_edf0d763-95bc-40d3-93a7-95b517162cfb']\n", - "inference> Here are the top 5 topics explained in the documentation:\n", - "\n", - "* What is LoRA and how does it work?\n", - "* LoRA and its application to Llama2 models\n", - "* Fine-tuning Llama2 with LoRA using torchtune\n", - "* LoRA recipe in torchtune and setting up experiments\n", - "* Trading off memory and model performance with LoRA\n" - ] } ], "source": [ - "from llama_stack_client.lib.agents.agent import Agent\n", - "from llama_stack_client.lib.agents.event_logger import EventLogger\n", - "from llama_stack_client.types.agent_create_params import AgentConfig\n", - "from llama_stack_client.types import Attachment\n", - "from termcolor import cprint\n", - "\n", - "urls = [\"chat.rst\", \"llama3.rst\", \"datasets.rst\", \"lora_finetune.rst\"]\n", - "attachments = [\n", - " Attachment(\n", - " content=f\"https://raw.githubusercontent.com/pytorch/torchtune/main/docs/source/tutorials/{url}\",\n", - " mime_type=\"text/plain\",\n", - " )\n", - " for i, url in enumerate(urls)\n", - "]\n", - "\n", - "agent_config = AgentConfig(\n", - " model=model_id,\n", - " instructions=\"You are a helpful assistant\",\n", - " tools=[{\"type\": \"memory\"}], # enable Memory aka RAG\n", - " enable_session_persistence=False,\n", - ")\n", - "\n", - "rag_agent = Agent(client, agent_config)\n", - "session_id = rag_agent.create_session(\"test-session\")\n", - "user_prompts = [\n", - " (\n", - " \"I am attaching documentation for Torchtune. Help me answer questions I will ask next.\",\n", - " attachments,\n", - " ),\n", - " (\n", - " \"What are the top 5 topics that were explained? Only list succinct bullet points.\",\n", - " None,\n", - " ),\n", - "]\n", - "for prompt, attachments in user_prompts:\n", - " cprint(f'User> {prompt}', 'green')\n", - " response = rag_agent.create_turn(\n", - " messages=[{\"role\": \"user\", \"content\": prompt}],\n", - " attachments=attachments,\n", - " session_id=session_id,\n", - " )\n", - " for log in EventLogger().log(response):\n", - " log.print()" + "from rich.pretty import pprint\n", + "for toolgroup in client.toolgroups.list():\n", + " pprint(toolgroup)" ] }, { @@ -1550,52 +1879,41 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "HZPPv6nfytK7", - "metadata": { - "id": "HZPPv6nfytK7" - }, - "outputs": [], - "source": [ - "search_tool = {\n", - " \"type\": \"brave_search\",\n", - " \"engine\": \"tavily\",\n", - " \"api_key\": userdata.get(\"TAVILY_SEARCH_API_KEY\")\n", - "}" - ] - }, - { - "cell_type": "code", - "execution_count": null, + "execution_count": 12, "id": "WS8Gu5b0APHs", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "WS8Gu5b0APHs", - "outputId": "48c3df89-4103-468a-f6f6-fc116d177380" + "outputId": "ec38efab-ca5b-478f-94b6-fd65a3cb3bb9" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "User> Hello\n", - "inference> Hello! How can I assist you today?\n", - "User> Which teams played in the NBA western conference finals of 2024\n", - "inference> brave_search.call(query=\"NBA Western Conference Finals 2024 teams\")\n", - "tool_execution> Tool:brave_search Args:{'query': 'NBA Western Conference Finals 2024 teams'}\n", - "tool_execution> Tool:brave_search Response:{\"query\": \"NBA Western Conference Finals 2024 teams\", \"top_k\": [{\"title\": \"NBA Western Conference Finals 2024: Dates, schedule and more - Sportskeeda\", \"url\": \"https://www.sportskeeda.com/basketball/news-nba-western-conference-finals-2024-dates-schedule-and-more\", \"content\": \"NBA Western Conference Finals 2024: Dates & Schedule The 2023-24 NBA Western Conference Finals will start on Wednesday, May 22. The Mavericks will face the team that wins in Game 7 between the\", \"score\": 0.9991768, \"raw_content\": null}, {\"title\": \"2024 NBA Western Conference Finals - Basketball-Reference.com\", \"url\": \"https://www.basketball-reference.com/playoffs/2024-nba-western-conference-finals-mavericks-vs-timberwolves.html\", \"content\": \"2024 NBA Western Conference Finals Mavericks vs. Timberwolves League Champion: Boston Celtics. Finals MVP: Jaylen Brown (20.8 / 5.4 / 5.0) 2024 Playoff Leaders: PTS: Luka Don\\u010di\\u0107 (635) TRB: Luka Don\\u010di\\u0107 (208) AST: Luka Don\\u010di\\u0107 (178) WS: Derrick White (2.9) More playoffs info\", \"score\": 0.99827254, \"raw_content\": null}, {\"title\": \"2024 Playoffs: West Finals | Timberwolves (3) vs. Mavericks (5) - NBA.com\", \"url\": \"https://www.nba.com/playoffs/2024/west-final\", \"content\": \"The Dallas Mavericks and Minnesota Timberwolves have advanced to the 2024 Western Conference Finals during the NBA playoffs.\", \"score\": 0.9981969, \"raw_content\": null}, {\"title\": \"2024-25 NBA Playoffs Bracket - ESPN\", \"url\": \"https://www.espn.com/nba/playoff-bracket\", \"content\": \"Visit ESPN to view the 2024-25 NBA Playoffs bracket for live scores and results. ... Teams. Odds. NBA Cup Bracket ... Western Conference. OKC wins series 4-0. 1. Thunder. 97. 8.\", \"score\": 0.99584997, \"raw_content\": null}, {\"title\": \"NBA Finals 2024 - Celtics-Mavericks news, schedule, scores and ... - ESPN\", \"url\": \"https://www.espn.com/nba/story/_/id/39943302/nba-playoffs-2024-conference-finals-news-scores-highlights\", \"content\": \"The Boston Celtics are the 2024 NBA Champions. ... Western Conference. Final 2023-24 NBA regular-season standings. Which team left standing has the most trips to the NBA Finals? Here is a look at\", \"score\": 0.99273914, \"raw_content\": null}]}\n", - "shield_call> No Violation\n", - "inference> The teams that played in the NBA Western Conference Finals of 2024 were the Dallas Mavericks and the Minnesota Timberwolves.\n" + "\u001b[32mUser> Hello\u001b[0m\n", + "\u001b[30m\u001b[0m\u001b[33minference> \u001b[0m\u001b[33mHello\u001b[0m\u001b[33m.\u001b[0m\u001b[33m How\u001b[0m\u001b[33m can\u001b[0m\u001b[33m I\u001b[0m\u001b[33m assist\u001b[0m\u001b[33m you\u001b[0m\u001b[33m today\u001b[0m\u001b[33m?\u001b[0m\u001b[97m\u001b[0m\n", + "\u001b[30m\u001b[0m\u001b[32mUser> Which teams played in the NBA western conference finals of 2024\u001b[0m\n", + "\u001b[30m\u001b[0m\u001b[33minference> \u001b[0m\u001b[36m\u001b[0m\u001b[36mbr\u001b[0m\u001b[36mave\u001b[0m\u001b[36m_search\u001b[0m\u001b[36m.call\u001b[0m\u001b[36m(query\u001b[0m\u001b[36m=\"\u001b[0m\u001b[36mN\u001b[0m\u001b[36mBA\u001b[0m\u001b[36m Western\u001b[0m\u001b[36m Conference\u001b[0m\u001b[36m Finals\u001b[0m\u001b[36m \u001b[0m\u001b[36m202\u001b[0m\u001b[36m4\u001b[0m\u001b[36m teams\u001b[0m\u001b[36m\")\u001b[0m\u001b[97m\u001b[0m\n", + "\u001b[32mtool_execution> Tool:brave_search Args:{'query': 'NBA Western Conference Finals 2024 teams'}\u001b[0m\n", + "\u001b[32mtool_execution> Tool:brave_search Response:{\"query\": \"NBA Western Conference Finals 2024 teams\", \"top_k\": [{\"title\": \"2024 NBA Western Conference Finals - Basketball-Reference.com\", \"url\": \"https://www.basketball-reference.com/playoffs/2024-nba-western-conference-finals-mavericks-vs-timberwolves.html\", \"content\": \"2024 NBA Western Conference Finals Mavericks vs. Timberwolves League Champion: Boston Celtics. Finals MVP: Jaylen Brown (20.8 / 5.4 / 5.0) 2024 Playoff Leaders: PTS: Luka Don\\u010di\\u0107 (635) TRB: Luka Don\\u010di\\u0107 (208) AST: Luka Don\\u010di\\u0107 (178) WS: Derrick White (2.9) More playoffs info\", \"score\": 0.9310187, \"raw_content\": null}, {\"title\": \"NBA Western Conference Finals 2024: Dates, schedule and more - Sportskeeda\", \"url\": \"https://www.sportskeeda.com/basketball/news-nba-western-conference-finals-2024-dates-schedule-and-more\", \"content\": \"NBA Western Conference Finals 2024: Dates & Schedule The 2023-24 NBA Western Conference Finals will start on Wednesday, May 22. The Mavericks will face the team that wins in Game 7 between the\", \"score\": 0.8914433, \"raw_content\": null}, {\"title\": \"2024 Playoffs: West Finals | Timberwolves (3) vs. Mavericks (5) - NBA.com\", \"url\": \"https://www.nba.com/playoffs/2024/west-final\", \"content\": \"The Dallas Mavericks and Minnesota Timberwolves have advanced to the 2024 Western Conference Finals during the NBA playoffs.\", \"score\": 0.8884594, \"raw_content\": null}, {\"title\": \"NBA Conference Finals Schedule: Full List of Games & Results\", \"url\": \"https://www.si.com/nba/nba-conference-finals-schedule-full-list-of-games-results\", \"content\": \"The 2024 NBA conference finals matchups are set. Here's the schedule for all the games. ... Western Conference First Round (1) Oklahoma City Thunder def. (8) New Orleans Pelicans in 4 games\", \"score\": 0.850382, \"raw_content\": null}, {\"title\": \"NBA Finals 2024 - Celtics-Mavericks news, schedule, scores and ... - ESPN\", \"url\": \"https://www.espn.com/nba/story/_/id/39943302/nba-playoffs-2024-conference-finals-news-scores-highlights\", \"content\": \"The Boston Celtics are the 2024 NBA Champions. ... Western Conference. Final 2023-24 NBA regular-season standings. Which team left standing has the most trips to the NBA Finals? Here is a look at\", \"score\": 0.8194462, \"raw_content\": null}]}\u001b[0m\n", + "\u001b[33minference> \u001b[0m\u001b[33mThe\u001b[0m\u001b[33m teams\u001b[0m\u001b[33m that\u001b[0m\u001b[33m played\u001b[0m\u001b[33m in\u001b[0m\u001b[33m the\u001b[0m\u001b[33m NBA\u001b[0m\u001b[33m Western\u001b[0m\u001b[33m Conference\u001b[0m\u001b[33m Finals\u001b[0m\u001b[33m of\u001b[0m\u001b[33m \u001b[0m\u001b[33m202\u001b[0m\u001b[33m4\u001b[0m\u001b[33m were\u001b[0m\u001b[33m the\u001b[0m\u001b[33m Dallas\u001b[0m\u001b[33m Mavericks\u001b[0m\u001b[33m and\u001b[0m\u001b[33m the\u001b[0m\u001b[33m Minnesota\u001b[0m\u001b[33m Timber\u001b[0m\u001b[33mw\u001b[0m\u001b[33molves\u001b[0m\u001b[33m.\u001b[0m\u001b[97m\u001b[0m\n", + "\u001b[30m\u001b[0m" ] } ], "source": [ + "from llama_stack_client.lib.agents.agent import Agent\n", + "from llama_stack_client.lib.agents.event_logger import EventLogger\n", + "from llama_stack_client.types.agent_create_params import AgentConfig\n", + "from termcolor import cprint\n", + "\n", "agent_config = AgentConfig(\n", " model=model_id,\n", " instructions=\"You are a helpful assistant\",\n", - " tools=[search_tool],\n", + " toolgroups=[\"builtin::websearch\"],\n", " input_shields=[],\n", " output_shields=[],\n", " enable_session_persistence=False,\n", @@ -1608,7 +1926,7 @@ "\n", "session_id = agent.create_session(\"test-session\")\n", "for prompt in user_prompts:\n", - " cprint(f'User> {prompt}', 'green')\n", + " cprint(f\"User> {prompt}\", \"green\")\n", " response = agent.create_turn(\n", " messages=[\n", " {\n", @@ -1622,6 +1940,206 @@ " log.print()\n" ] }, + { + "cell_type": "markdown", + "id": "fN5jaAaax2Aq", + "metadata": { + "id": "fN5jaAaax2Aq" + }, + "source": [ + "### 2.3. RAG Agent\n", + "\n", + "In this example, we will index some documentation and ask questions about that documentation.\n", + "\n", + "The tool we use is the memory tool. Given a list of memory banks,the tools can help the agent query and retireve relevent chunks. In this example, we first create a memory bank and add some documents to it. Then configure the agent to use the memory tool. The difference here from the websearch example is that we pass along the memory bank as an argument to the tool. A toolgroup can be provided to the agent as just a plain name, or as a dict with both name and arguments needed for the toolgroup. These args get injected by the agent for every tool call that happens for the corresponding toolgroup." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "GvLWltzZCNkg", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 351, + "referenced_widgets": [ + "edc4d84302f746d39a43e8107af6b67b", + "980292182c7144e194604c13ac544a26", + "8dee873065a047799a04e49ab791e449", + "29683ef34d5646c687118a2a0cdec6d4", + "3ec694106303491ea112a257309bc69c", + "288c9da81b3c4d80a4959753da973f58", + "cf453a1ed54645aba656f9a3f1461e69", + "ec747bd7c37c45298896c513634cd59a", + "5a620017a5384af1a056de687b2670db", + "8d370762fafd4d7887ff68ea8279d083", + "b6a0eb553b024a71b737ff47ca8f7633", + "2eff72cbd9bb4f1ca77213602caa9417", + "e82b5196209f4b9f919c7abb402a4504", + "fe34706489c14253a5015ff6332ec4e0", + "2574b07e4af24715aa89d048cc84e358", + "10bc8be68b5545fd8609824b02499ebf", + "d2473b7a6c5b4483981516af2fc59bde", + "4282ee7d947e426ba863df9970e82f3f", + "cfe6be8fd8254bc084a81b1d06e86ae1", + "1817f6732a5f44c7adc75a644b1acef2", + "7551b282ef3a4387a801637de2d5c76e", + "69e5263c812c4542a9e5c31fefaa37fe", + "7cc356ed20e94401b72a0e138ad0f5df", + "acd39276db17439798a97abc56460b0f", + "bda474c3b8184597a6a9bc6da0672a50", + "20a66f9de4ed41c7ac9a8e817898ed9e", + "e662ba10fbae49d9b66172125dfc0717", + "d452b32c54e14e41a17fd7d51862ba8e", + "d1f8f4568a444248b69022d58e3f1af0", + "0c2e30d78c234b1b8098d879442d3bac", + "9bb8bf12010f42b2b17c10c7ccaa7bf8", + "2b2046db907349798e3ae774c15b25d2", + "3c18f449359f422f950543bd976fe323", + "472b1acc4c5a4c48b2ec62be42d1830c", + "44e34588d6854737b0fb14b4b6a62a95", + "03402ad03418435ca7a550e3246cd300", + "811f115733b14ab4b242a8b11526016c", + "e61fdef1dc4b4d809168c0b441b0e6ac", + "631c9a95127244c79875c829a7637df6", + "d25492ad867141bfa8d957d2464b8639", + "9df914248c214597bed7d7980c7a0afe", + "4709067f3f554b93b3ef35e3f58cbf85", + "02baf670942347d69c290452de8641e4", + "7611cfc7965649ba88ca57c1a9f9ccf3", + "15ae23892b634a9f821a8fcee14e500b", + "b28d46c2ecdd46b9b3f2da871afbf1cb", + "4b83e3caa8ec47169dca04ee9599adeb", + "c83c23161674484e81f0db9856c23eb6", + "3ded85d9c34246e88f8ce693eb8025e5", + "0ac8e976a32c4f5989392b8088546e00", + "ed4b0035752546cc81688a7a77ba27c0", + "269b1ad9dc7b4ebb94d7364c75f3f324", + "2256ddab0ae1408abb10ba211a08f794", + "42335bcbc6ee40a79d36c5159cc7da06", + "cf694e1b797246b096ae588973dc985f", + "3e764c00c08942caa2ccb6b92ee60a4e", + "af6680f2e60e476d8487aea98a23b84e", + "c26a9d456e904b2b900bf5e0a5964a0d", + "5a3e0b5ae83143329de6507f9bcf83e0", + "3c9bc5588765436da4f1fee2d893cafd" + ] + }, + "id": "GvLWltzZCNkg", + "outputId": "ef5f3ec4-edaf-4705-fb1b-b86659d7143c" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Batches: 100%|██████████| 1/1 [00:00<00:00, 1.83it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[32mUser> What are the top 5 topics that were explained? Only list succinct bullet points.\u001b[0m\n", + "\u001b[30m\u001b[0m" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "Batches: 100%|██████████| 1/1 [00:00<00:00, 7.28it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[32mtool_execution> Tool:query_from_memory Args:{}\u001b[0m\n", + "\u001b[36mtool_execution> fetched 10913 bytes from memory\u001b[0m\n", + "\u001b[33minference> \u001b[0m" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mHere\u001b[0m\u001b[33m are\u001b[0m\u001b[33m the\u001b[0m\u001b[33m top\u001b[0m\u001b[33m \u001b[0m\u001b[33m5\u001b[0m\u001b[33m topics\u001b[0m\u001b[33m that\u001b[0m\u001b[33m were\u001b[0m\u001b[33m explained\u001b[0m\u001b[33m:\n", + "\n", + "\u001b[0m\u001b[33m•\u001b[0m\u001b[33m Token\u001b[0m\u001b[33mizing\u001b[0m\u001b[33m prompt\u001b[0m\u001b[33m templates\u001b[0m\u001b[33m and\u001b[0m\u001b[33m special\u001b[0m\u001b[33m tokens\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33m•\u001b[0m\u001b[33m Fine\u001b[0m\u001b[33m-t\u001b[0m\u001b[33muning\u001b[0m\u001b[33m on\u001b[0m\u001b[33m a\u001b[0m\u001b[33m custom\u001b[0m\u001b[33m chat\u001b[0m\u001b[33m dataset\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33m•\u001b[0m\u001b[33m Using\u001b[0m\u001b[33m the\u001b[0m\u001b[33m L\u001b[0m\u001b[33mlama\u001b[0m\u001b[33m2\u001b[0m\u001b[33mChat\u001b[0m\u001b[33mTemplate\u001b[0m\u001b[33m class\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33m•\u001b[0m\u001b[33m Formatting\u001b[0m\u001b[33m messages\u001b[0m\u001b[33m with\u001b[0m\u001b[33m the\u001b[0m\u001b[33m L\u001b[0m\u001b[33mlama\u001b[0m\u001b[33m2\u001b[0m\u001b[33mChat\u001b[0m\u001b[33mTemplate\u001b[0m\u001b[33m class\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33m•\u001b[0m\u001b[33m Creating\u001b[0m\u001b[33m a\u001b[0m\u001b[33m custom\u001b[0m\u001b[33m dataset\u001b[0m\u001b[33m for\u001b[0m\u001b[33m fine\u001b[0m\u001b[33m-t\u001b[0m\u001b[33muning\u001b[0m\u001b[33m L\u001b[0m\u001b[33mlama\u001b[0m\u001b[33m3\u001b[0m\u001b[97m\u001b[0m\n", + "\u001b[30m\u001b[0m" + ] + } + ], + "source": [ + "from llama_stack_client.lib.agents.agent import Agent\n", + "from llama_stack_client.lib.agents.event_logger import EventLogger\n", + "from llama_stack_client.types.agent_create_params import AgentConfig\n", + "from termcolor import cprint\n", + "from llama_stack_client.types import Document\n", + "\n", + "urls = [\"chat.rst\", \"llama3.rst\", \"datasets.rst\", \"lora_finetune.rst\"]\n", + "documents = [\n", + " Document(\n", + " document_id=f\"num-{i}\",\n", + " content=f\"https://raw.githubusercontent.com/pytorch/torchtune/main/docs/source/tutorials/{url}\",\n", + " mime_type=\"text/plain\",\n", + " metadata={},\n", + " )\n", + " for i, url in enumerate(urls)\n", + "]\n", + "\n", + "vector_db_id = \"test-vector-db\"\n", + "client.vector_dbs.register(\n", + " vector_db_id=vector_db_id,\n", + " embedding_model=\"all-MiniLM-L6-v2\",\n", + " embedding_dimension=384,\n", + ")\n", + "client.tool_runtime.rag_tool.insert(\n", + " documents=documents,\n", + " vector_db_id=vector_db_id,\n", + " chunk_size_in_tokens=512,\n", + ")\n", + "agent_config = AgentConfig(\n", + " model=model_id,\n", + " instructions=\"You are a helpful assistant\",\n", + " enable_session_persistence=False,\n", + " toolgroups = [\n", + " {\n", + " \"name\": \"builtin::rag\",\n", + " \"args\" : {\n", + " \"vector_db_ids\": [vector_db_id],\n", + " }\n", + " }\n", + " ],\n", + ")\n", + "rag_agent = Agent(client, agent_config)\n", + "session_id = rag_agent.create_session(\"test-session\")\n", + "user_prompts = [\n", + " \"What are the top 5 topics that were explained? Only list succinct bullet points.\",\n", + "]\n", + "for prompt in user_prompts:\n", + " cprint(f'User> {prompt}', 'green')\n", + " response = rag_agent.create_turn(\n", + " messages=[{\"role\": \"user\", \"content\": prompt}],\n", + " session_id=session_id,\n", + " )\n", + " for log in EventLogger().log(response):\n", + " log.print()" + ] + }, { "cell_type": "markdown", "id": "yRzRwu8qxyl0", @@ -1629,14 +2147,14 @@ "id": "yRzRwu8qxyl0" }, "source": [ - "### 2.3. Code Execution Agent\n", + "### 2.4. Code Execution Agent\n", "\n", "In this example, we will show how multiple tools can be called by the model - including web search and code execution. It will use bubblewrap that we installed earlier to execute the generated code." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "id": "GvVRuhO-GOov", "metadata": { "colab": { @@ -1644,160 +2162,135 @@ }, "collapsed": true, "id": "GvVRuhO-GOov", - "outputId": "cb988aa9-568b-4966-d500-575b7b24578f" + "outputId": "39395e26-bb7d-4616-d51d-036c8bf41427" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "User> ('Here is a csv, can you describe it ?', [Attachment(content='https://raw.githubusercontent.com/meta-llama/llama-stack-apps/main/examples/resources/inflation.csv', mime_type='test/csv')])\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:httpx:HTTP Request: GET https://raw.githubusercontent.com/meta-llama/llama-stack-apps/main/examples/resources/inflation.csv \"HTTP/1.1 200 OK\"\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "inference> import pandas as pd\n", - "\n", - "# Read the CSV file\n", - "df = pd.read_csv('/tmp/tmpco0s0o4_/LOdZoVp1inflation.csv')\n", - "\n", - "# Describe the CSV\n", - "print(df.describe())\n", - "tool_execution> Tool:code_interpreter Args:{'code': \"import pandas as pd\\n\\n# Read the CSV file\\ndf = pd.read_csv('/tmp/tmpco0s0o4_/LOdZoVp1inflation.csv')\\n\\n# Describe the CSV\\nprint(df.describe())\"}\n", - "tool_execution> Tool:code_interpreter Response:completed\n", + "\u001b[32mUser> Here is a csv, can you describe it?\u001b[0m\n", + "\u001b[30m\u001b[0m\u001b[33minference> \u001b[0m\u001b[36m\u001b[0m\u001b[36mimport\u001b[0m\u001b[36m pandas\u001b[0m\u001b[36m as\u001b[0m\u001b[36m pd\u001b[0m\u001b[36m\n", + "\u001b[0m\u001b[36m#\u001b[0m\u001b[36m Load\u001b[0m\u001b[36m data\u001b[0m\u001b[36m\n", + "\u001b[0m\u001b[36mdf\u001b[0m\u001b[36m =\u001b[0m\u001b[36m pd\u001b[0m\u001b[36m.read\u001b[0m\u001b[36m_csv\u001b[0m\u001b[36m('/\u001b[0m\u001b[36mvar\u001b[0m\u001b[36m/f\u001b[0m\u001b[36molders\u001b[0m\u001b[36m/m\u001b[0m\u001b[36mj\u001b[0m\u001b[36m/t\u001b[0m\u001b[36m_st\u001b[0m\u001b[36mv\u001b[0m\u001b[36m1\u001b[0m\u001b[36mys\u001b[0m\u001b[36m763\u001b[0m\u001b[36m7\u001b[0m\u001b[36mv\u001b[0m\u001b[36mq\u001b[0m\u001b[36mf\u001b[0m\u001b[36m2\u001b[0m\u001b[36m_b\u001b[0m\u001b[36m4\u001b[0m\u001b[36my\u001b[0m\u001b[36mf\u001b[0m\u001b[36m67\u001b[0m\u001b[36mm\u001b[0m\u001b[36m000\u001b[0m\u001b[36m0\u001b[0m\u001b[36mgn\u001b[0m\u001b[36m/T\u001b[0m\u001b[36m/tmp\u001b[0m\u001b[36mq\u001b[0m\u001b[36m2\u001b[0m\u001b[36mw\u001b[0m\u001b[36mjj\u001b[0m\u001b[36mmg\u001b[0m\u001b[36mf\u001b[0m\u001b[36m/s\u001b[0m\u001b[36mQ\u001b[0m\u001b[36mAm\u001b[0m\u001b[36muk\u001b[0m\u001b[36mV\u001b[0m\u001b[36mbin\u001b[0m\u001b[36mflation\u001b[0m\u001b[36m.csv\u001b[0m\u001b[36m')\n", + "\u001b[0m\u001b[36m#\u001b[0m\u001b[36m Set\u001b[0m\u001b[36m options\u001b[0m\u001b[36m\n", + "\u001b[0m\u001b[36mpd\u001b[0m\u001b[36m.set\u001b[0m\u001b[36m_option\u001b[0m\u001b[36m('\u001b[0m\u001b[36mdisplay\u001b[0m\u001b[36m.max\u001b[0m\u001b[36m_columns\u001b[0m\u001b[36m',\u001b[0m\u001b[36m None\u001b[0m\u001b[36m)\n", + "\u001b[0m\u001b[36mpd\u001b[0m\u001b[36m.set\u001b[0m\u001b[36m_option\u001b[0m\u001b[36m('\u001b[0m\u001b[36mdisplay\u001b[0m\u001b[36m.max\u001b[0m\u001b[36m_rows\u001b[0m\u001b[36m',\u001b[0m\u001b[36m None\u001b[0m\u001b[36m)\n", + "\u001b[0m\u001b[36m#\u001b[0m\u001b[36m Describe\u001b[0m\u001b[36m the\u001b[0m\u001b[36m data\u001b[0m\u001b[36m\n", + "\u001b[0m\u001b[36mprint\u001b[0m\u001b[36m(df\u001b[0m\u001b[36m.describe\u001b[0m\u001b[36m())\u001b[0m\u001b[97m\u001b[0m\n", + "\u001b[32mtool_execution> Tool:code_interpreter Args:{'code': \"import pandas as pd\\n# Load data\\ndf = pd.read_csv('/var/folders/mj/t_stv1ys7637vqf2_b4yf67m0000gn/T/tmpq2wjjmgf/sQAmukVbinflation.csv')\\n# Set options\\npd.set_option('display.max_columns', None)\\npd.set_option('display.max_rows', None)\\n# Describe the data\\nprint(df.describe())\"}\u001b[0m\n", + "\u001b[32mtool_execution> Tool:code_interpreter Response:error\n", "[stdout]\n", - "Year Jan Feb Mar ... Sep Oct Nov Dec\n", - "count 10.00000 10.000000 10.000000 10.000000 ... 10.000000 10.000000 10.000000 10.000000\n", - "mean 2018.50000 2.700000 2.730000 2.760000 ... 2.850000 2.850000 2.850000 2.890000\n", - "std 3.02765 1.667999 1.743591 1.757018 ... 1.593912 1.577093 1.551523 1.569466\n", - "min 2014.00000 1.400000 1.300000 1.600000 ... 1.700000 1.600000 1.600000 1.600000\n", - "25% 2016.25000 1.650000 1.725000 1.850000 ... 1.750000 1.825000 1.775000 1.875000\n", - "50% 2018.50000 2.200000 2.150000 2.050000 ... 2.200000 2.100000 2.150000 2.200000\n", - "75% 2020.75000 2.300000 2.375000 2.175000 ... 3.600000 3.575000 3.575000 3.500000\n", - "max 2023.00000 6.000000 6.400000 6.500000 ... 6.600000 6.300000 6.000000 5.700000\n", - "\n", - "[8 rows x 13 columns]\n", + "[Errno 2] No such file or directory: 'bwrap'\n", "[/stdout]\n", - "shield_call> No Violation\n", - "inference> The CSV file appears to be a dataset with 10 rows and 13 columns. The columns represent various economic indicators, such as inflation rates for each month from January to December, as well as year (yearly inflation rate).\n", + "[stderr]\n", + "[Errno 2] No such file or directory: 'bwrap'\n", + "[/stderr]\u001b[0m\n", + "\u001b[33minference> \u001b[0m\u001b[33mIt\u001b[0m\u001b[33m seems\u001b[0m\u001b[33m that\u001b[0m\u001b[33m there\u001b[0m\u001b[33m was\u001b[0m\u001b[33m an\u001b[0m\u001b[33m issue\u001b[0m\u001b[33m with\u001b[0m\u001b[33m accessing\u001b[0m\u001b[33m the\u001b[0m\u001b[33m file\u001b[0m\u001b[33m.\u001b[0m\u001b[33m I\u001b[0m\u001b[33m'm\u001b[0m\u001b[33m a\u001b[0m\u001b[33m large\u001b[0m\u001b[33m language\u001b[0m\u001b[33m model\u001b[0m\u001b[33m,\u001b[0m\u001b[33m I\u001b[0m\u001b[33m don\u001b[0m\u001b[33m't\u001b[0m\u001b[33m have\u001b[0m\u001b[33m the\u001b[0m\u001b[33m ability\u001b[0m\u001b[33m to\u001b[0m\u001b[33m access\u001b[0m\u001b[33m or\u001b[0m\u001b[33m read\u001b[0m\u001b[33m files\u001b[0m\u001b[33m from\u001b[0m\u001b[33m your\u001b[0m\u001b[33m local\u001b[0m\u001b[33m system\u001b[0m\u001b[33m.\u001b[0m\u001b[33m However\u001b[0m\u001b[33m,\u001b[0m\u001b[33m I\u001b[0m\u001b[33m can\u001b[0m\u001b[33m guide\u001b[0m\u001b[33m you\u001b[0m\u001b[33m on\u001b[0m\u001b[33m how\u001b[0m\u001b[33m to\u001b[0m\u001b[33m describe\u001b[0m\u001b[33m a\u001b[0m\u001b[33m CSV\u001b[0m\u001b[33m file\u001b[0m\u001b[33m using\u001b[0m\u001b[33m Python\u001b[0m\u001b[33m's\u001b[0m\u001b[33m pandas\u001b[0m\u001b[33m library\u001b[0m\u001b[33m.\n", "\n", - "Here is a brief description of the data:\n", + "\u001b[0m\u001b[33mTo\u001b[0m\u001b[33m describe\u001b[0m\u001b[33m a\u001b[0m\u001b[33m CSV\u001b[0m\u001b[33m file\u001b[0m\u001b[33m,\u001b[0m\u001b[33m you\u001b[0m\u001b[33m can\u001b[0m\u001b[33m use\u001b[0m\u001b[33m the\u001b[0m\u001b[33m `\u001b[0m\u001b[33mp\u001b[0m\u001b[33mandas\u001b[0m\u001b[33m`\u001b[0m\u001b[33m library\u001b[0m\u001b[33m in\u001b[0m\u001b[33m Python\u001b[0m\u001b[33m.\u001b[0m\u001b[33m Here\u001b[0m\u001b[33m's\u001b[0m\u001b[33m a\u001b[0m\u001b[33m step\u001b[0m\u001b[33m-by\u001b[0m\u001b[33m-step\u001b[0m\u001b[33m guide\u001b[0m\u001b[33m:\n", "\n", - "* The `Year` column contains the year for which the inflation rate is reported.\n", - "* The `Jan`, `Feb`, `Mar`, etc. columns contain the inflation rate for each month (January to December).\n", - "* The `count` column is the count of non-null values in each column.\n", - "* The `mean` column is the mean of the non-null values in each column.\n", - "* The `std` column is the standard deviation of the non-null values in each column.\n", - "* The `min` column is the minimum value in each column.\n", - "* The `25%` column is the 25th percentile (25th percentile) of the non-null values in each column.\n", - "* The `50%` column is the 50th percentile (50th percentile) of the non-null values in each column.\n", - "* The `75%` column is the 75th percentile (75th percentile) of the non-null values in each column.\n", - "* The `max` column is the maximum value in each column.\n", + "\u001b[0m\u001b[33m1\u001b[0m\u001b[33m.\u001b[0m\u001b[33m Install\u001b[0m\u001b[33m the\u001b[0m\u001b[33m `\u001b[0m\u001b[33mp\u001b[0m\u001b[33mandas\u001b[0m\u001b[33m`\u001b[0m\u001b[33m library\u001b[0m\u001b[33m if\u001b[0m\u001b[33m you\u001b[0m\u001b[33m haven\u001b[0m\u001b[33m't\u001b[0m\u001b[33m already\u001b[0m\u001b[33m:\u001b[0m\u001b[33m `\u001b[0m\u001b[33mpip\u001b[0m\u001b[33m install\u001b[0m\u001b[33m pandas\u001b[0m\u001b[33m`\n", + "\u001b[0m\u001b[33m2\u001b[0m\u001b[33m.\u001b[0m\u001b[33m Import\u001b[0m\u001b[33m the\u001b[0m\u001b[33m `\u001b[0m\u001b[33mp\u001b[0m\u001b[33mandas\u001b[0m\u001b[33m`\u001b[0m\u001b[33m library\u001b[0m\u001b[33m:\u001b[0m\u001b[33m `\u001b[0m\u001b[33mimport\u001b[0m\u001b[33m pandas\u001b[0m\u001b[33m as\u001b[0m\u001b[33m pd\u001b[0m\u001b[33m`\n", + "\u001b[0m\u001b[33m3\u001b[0m\u001b[33m.\u001b[0m\u001b[33m Load\u001b[0m\u001b[33m the\u001b[0m\u001b[33m CSV\u001b[0m\u001b[33m file\u001b[0m\u001b[33m into\u001b[0m\u001b[33m a\u001b[0m\u001b[33m DataFrame\u001b[0m\u001b[33m:\u001b[0m\u001b[33m `\u001b[0m\u001b[33mdf\u001b[0m\u001b[33m =\u001b[0m\u001b[33m pd\u001b[0m\u001b[33m.read\u001b[0m\u001b[33m_csv\u001b[0m\u001b[33m('\u001b[0m\u001b[33myour\u001b[0m\u001b[33m_file\u001b[0m\u001b[33m.csv\u001b[0m\u001b[33m')\u001b[0m\u001b[33m`\n", + "\u001b[0m\u001b[33m4\u001b[0m\u001b[33m.\u001b[0m\u001b[33m Use\u001b[0m\u001b[33m the\u001b[0m\u001b[33m `\u001b[0m\u001b[33mdescribe\u001b[0m\u001b[33m()`\u001b[0m\u001b[33m method\u001b[0m\u001b[33m to\u001b[0m\u001b[33m get\u001b[0m\u001b[33m a\u001b[0m\u001b[33m summary\u001b[0m\u001b[33m of\u001b[0m\u001b[33m the\u001b[0m\u001b[33m data\u001b[0m\u001b[33m:\u001b[0m\u001b[33m `\u001b[0m\u001b[33mprint\u001b[0m\u001b[33m(df\u001b[0m\u001b[33m.describe\u001b[0m\u001b[33m())\u001b[0m\u001b[33m`\n", "\n", - "This dataset could be used for various applications, such as analyzing historical inflation rates, forecasting future inflation rates, or comparing inflation rates across different months or years.\n", - "User> ('Which year ended with the highest inflation ?', None)\n", - "inference> According to the data, the year with the highest inflation was 2023. The inflation rate for 2023 is 6.600%.\n", - "User> ('What macro economic situations that led to such high inflation in that period?', None)\n", - "inference> The high inflation rate in 2023 is likely attributed to a combination of macroeconomic factors, including:\n", + "\u001b[0m\u001b[33mThis\u001b[0m\u001b[33m will\u001b[0m\u001b[33m give\u001b[0m\u001b[33m you\u001b[0m\u001b[33m a\u001b[0m\u001b[33m summary\u001b[0m\u001b[33m of\u001b[0m\u001b[33m the\u001b[0m\u001b[33m data\u001b[0m\u001b[33m,\u001b[0m\u001b[33m including\u001b[0m\u001b[33m the\u001b[0m\u001b[33m count\u001b[0m\u001b[33m,\u001b[0m\u001b[33m mean\u001b[0m\u001b[33m,\u001b[0m\u001b[33m standard\u001b[0m\u001b[33m deviation\u001b[0m\u001b[33m,\u001b[0m\u001b[33m minimum\u001b[0m\u001b[33m,\u001b[0m\u001b[33m \u001b[0m\u001b[33m25\u001b[0m\u001b[33mth\u001b[0m\u001b[33m percentile\u001b[0m\u001b[33m,\u001b[0m\u001b[33m \u001b[0m\u001b[33m50\u001b[0m\u001b[33mth\u001b[0m\u001b[33m percentile\u001b[0m\u001b[33m,\u001b[0m\u001b[33m \u001b[0m\u001b[33m75\u001b[0m\u001b[33mth\u001b[0m\u001b[33m percentile\u001b[0m\u001b[33m,\u001b[0m\u001b[33m and\u001b[0m\u001b[33m maximum\u001b[0m\u001b[33m for\u001b[0m\u001b[33m each\u001b[0m\u001b[33m column\u001b[0m\u001b[33m.\n", "\n", - "1. **Supply chain disruptions**: The COVID-19 pandemic and subsequent lockdowns led to supply chain disruptions, resulting in shortages and price increases for various goods and services.\n", - "2. **Economic growth**: The rapid economic growth in the preceding years created demand for goods and services, leading to higher production costs and, subsequently, higher prices.\n", - "3. **Monetary policy**: The central bank's easy-money policies, such as quantitative easing and low interest rates, increased the money supply and led to inflationary pressures.\n", - "4. **Commodity price shocks**: Increases in global commodity prices, such as oil and food prices, contributed to higher production costs and inflation.\n", - "5. **Labor market tightness**: The labor market has been tight, leading to higher wages and, subsequently, higher production costs, which have been passed on to consumers.\n", - "6. **Trade wars and tariffs**: The ongoing trade tensions and tariffs imposed by various countries have disrupted global supply chains, leading to higher prices for imported goods.\n", - "7. **Climate change and extreme weather events**: The increasing frequency and severity of extreme weather events, such as heatwaves and droughts, have disrupted agricultural production and supply chains.\n", - "8. **Currency devaluation**: A devaluation of the currency can make imports more expensive, leading to higher inflation.\n", - "9. **Government spending and fiscal policy**: Government spending and fiscal policy decisions, such as tax cuts and increased government spending, can inject more money into the economy, leading to inflation.\n", - "10. **Monetary policy mistakes**: Mistakes in monetary policy, such as premature interest rate hikes or overly aggressive quantitative easing, can lead to inflationary pressures.\n", + "\u001b[0m\u001b[33mIf\u001b[0m\u001b[33m you\u001b[0m\u001b[33m provide\u001b[0m\u001b[33m the\u001b[0m\u001b[33m contents\u001b[0m\u001b[33m of\u001b[0m\u001b[33m the\u001b[0m\u001b[33m CSV\u001b[0m\u001b[33m file\u001b[0m\u001b[33m,\u001b[0m\u001b[33m I\u001b[0m\u001b[33m can\u001b[0m\u001b[33m help\u001b[0m\u001b[33m you\u001b[0m\u001b[33m describe\u001b[0m\u001b[33m it\u001b[0m\u001b[33m.\u001b[0m\u001b[97m\u001b[0m\n", + "\u001b[30m\u001b[0m\u001b[32mUser> Plot average yearly inflation as a time series\u001b[0m\n", + "\u001b[30m\u001b[0m\u001b[33minference> \u001b[0m\u001b[36m\u001b[0m\u001b[36mimport\u001b[0m\u001b[36m pandas\u001b[0m\u001b[36m as\u001b[0m\u001b[36m pd\u001b[0m\u001b[36m\n", + "\u001b[0m\u001b[36mimport\u001b[0m\u001b[36m matplotlib\u001b[0m\u001b[36m.pyplot\u001b[0m\u001b[36m as\u001b[0m\u001b[36m plt\u001b[0m\u001b[36m\n", "\n", - "It's worth noting that the specific factors contributing to the high inflation rate in 2023 may vary depending on the region, country, or even specific economy.\n", - "User> ('Plot average yearly inflation as a time series', None)\n", - "inference> import pandas as pd\n", - "import matplotlib.pyplot as plt\n", + "\u001b[0m\u001b[36m#\u001b[0m\u001b[36m Load\u001b[0m\u001b[36m the\u001b[0m\u001b[36m data\u001b[0m\u001b[36m\n", + "\u001b[0m\u001b[36mdf\u001b[0m\u001b[36m =\u001b[0m\u001b[36m pd\u001b[0m\u001b[36m.read\u001b[0m\u001b[36m_csv\u001b[0m\u001b[36m('/\u001b[0m\u001b[36mvar\u001b[0m\u001b[36m/f\u001b[0m\u001b[36molders\u001b[0m\u001b[36m/m\u001b[0m\u001b[36mj\u001b[0m\u001b[36m/t\u001b[0m\u001b[36m_st\u001b[0m\u001b[36mv\u001b[0m\u001b[36m1\u001b[0m\u001b[36mys\u001b[0m\u001b[36m763\u001b[0m\u001b[36m7\u001b[0m\u001b[36mv\u001b[0m\u001b[36mq\u001b[0m\u001b[36mf\u001b[0m\u001b[36m2\u001b[0m\u001b[36m_b\u001b[0m\u001b[36m4\u001b[0m\u001b[36my\u001b[0m\u001b[36mf\u001b[0m\u001b[36m67\u001b[0m\u001b[36mm\u001b[0m\u001b[36m000\u001b[0m\u001b[36m0\u001b[0m\u001b[36mgn\u001b[0m\u001b[36m/T\u001b[0m\u001b[36m/tmp\u001b[0m\u001b[36mq\u001b[0m\u001b[36m2\u001b[0m\u001b[36mw\u001b[0m\u001b[36mjj\u001b[0m\u001b[36mmg\u001b[0m\u001b[36mf\u001b[0m\u001b[36m/s\u001b[0m\u001b[36mQ\u001b[0m\u001b[36mAm\u001b[0m\u001b[36muk\u001b[0m\u001b[36mV\u001b[0m\u001b[36mbin\u001b[0m\u001b[36mflation\u001b[0m\u001b[36m.csv\u001b[0m\u001b[36m')\n", "\n", - "# Read the CSV file\n", - "df = pd.read_csv('/tmp/tmpco0s0o4_/LOdZoVp1inflation.csv')\n", + "\u001b[0m\u001b[36m#\u001b[0m\u001b[36m Convert\u001b[0m\u001b[36m the\u001b[0m\u001b[36m '\u001b[0m\u001b[36myear\u001b[0m\u001b[36m'\u001b[0m\u001b[36m column\u001b[0m\u001b[36m to\u001b[0m\u001b[36m datetime\u001b[0m\u001b[36m\n", + "\u001b[0m\u001b[36mdf\u001b[0m\u001b[36m['\u001b[0m\u001b[36myear\u001b[0m\u001b[36m']\u001b[0m\u001b[36m =\u001b[0m\u001b[36m pd\u001b[0m\u001b[36m.to\u001b[0m\u001b[36m_datetime\u001b[0m\u001b[36m(df\u001b[0m\u001b[36m['\u001b[0m\u001b[36myear\u001b[0m\u001b[36m'])\n", "\n", - "# Extract the year and inflation rate from the CSV file\n", - "df['Year'] = pd.to_datetime(df['Year'], format='%Y')\n", - "df = df.rename(columns={'Jan': 'Jan Rate', 'Feb': 'Feb Rate', 'Mar': 'Mar Rate', 'Apr': 'Apr Rate', 'May': 'May Rate', 'Jun': 'Jun Rate', 'Jul': 'Jul Rate', 'Aug': 'Aug Rate', 'Sep': 'Sep Rate', 'Oct': 'Oct Rate', 'Nov': 'Nov Rate', 'Dec': 'Dec Rate'})\n", + "\u001b[0m\u001b[36m#\u001b[0m\u001b[36m Group\u001b[0m\u001b[36m by\u001b[0m\u001b[36m year\u001b[0m\u001b[36m and\u001b[0m\u001b[36m calculate\u001b[0m\u001b[36m the\u001b[0m\u001b[36m average\u001b[0m\u001b[36m inflation\u001b[0m\u001b[36m\n", + "\u001b[0m\u001b[36maverage\u001b[0m\u001b[36m_in\u001b[0m\u001b[36mflation\u001b[0m\u001b[36m =\u001b[0m\u001b[36m df\u001b[0m\u001b[36m.groupby\u001b[0m\u001b[36m('\u001b[0m\u001b[36myear\u001b[0m\u001b[36m')['\u001b[0m\u001b[36min\u001b[0m\u001b[36mflation\u001b[0m\u001b[36m'].\u001b[0m\u001b[36mmean\u001b[0m\u001b[36m()\n", "\n", - "# Calculate the average yearly inflation rate\n", - "df['Yearly Inflation'] = df[['Jan Rate', 'Feb Rate', 'Mar Rate', 'Apr Rate', 'May Rate', 'Jun Rate', 'Jul Rate', 'Aug Rate', 'Sep Rate', 'Oct Rate', 'Nov Rate', 'Dec Rate']].mean(axis=1)\n", + "\u001b[0m\u001b[36m#\u001b[0m\u001b[36m Plot\u001b[0m\u001b[36m the\u001b[0m\u001b[36m average\u001b[0m\u001b[36m yearly\u001b[0m\u001b[36m inflation\u001b[0m\u001b[36m as\u001b[0m\u001b[36m a\u001b[0m\u001b[36m time\u001b[0m\u001b[36m series\u001b[0m\u001b[36m\n", + "\u001b[0m\u001b[36mplt\u001b[0m\u001b[36m.figure\u001b[0m\u001b[36m(figsize\u001b[0m\u001b[36m=(\u001b[0m\u001b[36m10\u001b[0m\u001b[36m,\u001b[0m\u001b[36m6\u001b[0m\u001b[36m))\n", + "\u001b[0m\u001b[36mplt\u001b[0m\u001b[36m.plot\u001b[0m\u001b[36m(\u001b[0m\u001b[36maverage\u001b[0m\u001b[36m_in\u001b[0m\u001b[36mflation\u001b[0m\u001b[36m.index\u001b[0m\u001b[36m,\u001b[0m\u001b[36m average\u001b[0m\u001b[36m_in\u001b[0m\u001b[36mflation\u001b[0m\u001b[36m.values\u001b[0m\u001b[36m,\u001b[0m\u001b[36m marker\u001b[0m\u001b[36m='\u001b[0m\u001b[36mo\u001b[0m\u001b[36m')\n", + "\u001b[0m\u001b[36mplt\u001b[0m\u001b[36m.title\u001b[0m\u001b[36m('\u001b[0m\u001b[36mAverage\u001b[0m\u001b[36m Year\u001b[0m\u001b[36mly\u001b[0m\u001b[36m In\u001b[0m\u001b[36mflation\u001b[0m\u001b[36m')\n", + "\u001b[0m\u001b[36mplt\u001b[0m\u001b[36m.xlabel\u001b[0m\u001b[36m('\u001b[0m\u001b[36mYear\u001b[0m\u001b[36m')\n", + "\u001b[0m\u001b[36mplt\u001b[0m\u001b[36m.ylabel\u001b[0m\u001b[36m('\u001b[0m\u001b[36mAverage\u001b[0m\u001b[36m In\u001b[0m\u001b[36mflation\u001b[0m\u001b[36m')\n", + "\u001b[0m\u001b[36mplt\u001b[0m\u001b[36m.grid\u001b[0m\u001b[36m(True\u001b[0m\u001b[36m)\n", + "\u001b[0m\u001b[36mplt\u001b[0m\u001b[36m.show\u001b[0m\u001b[36m()\u001b[0m\u001b[97m\u001b[0m\n", + "\u001b[32mtool_execution> Tool:code_interpreter Args:{'code': \"import pandas as pd\\nimport matplotlib.pyplot as plt\\n\\n# Load the data\\ndf = pd.read_csv('/var/folders/mj/t_stv1ys7637vqf2_b4yf67m0000gn/T/tmpq2wjjmgf/sQAmukVbinflation.csv')\\n\\n# Convert the 'year' column to datetime\\ndf['year'] = pd.to_datetime(df['year'])\\n\\n# Group by year and calculate the average inflation\\naverage_inflation = df.groupby('year')['inflation'].mean()\\n\\n# Plot the average yearly inflation as a time series\\nplt.figure(figsize=(10,6))\\nplt.plot(average_inflation.index, average_inflation.values, marker='o')\\nplt.title('Average Yearly Inflation')\\nplt.xlabel('Year')\\nplt.ylabel('Average Inflation')\\nplt.grid(True)\\nplt.show()\"}\u001b[0m\n", + "\u001b[32mtool_execution> Tool:code_interpreter Response:error\n", + "[stdout]\n", + "[Errno 2] No such file or directory: 'bwrap'\n", + "[/stdout]\n", + "[stderr]\n", + "[Errno 2] No such file or directory: 'bwrap'\n", + "[/stderr]\u001b[0m\n", + "\u001b[33minference> \u001b[0m\u001b[33mIt\u001b[0m\u001b[33m seems\u001b[0m\u001b[33m that\u001b[0m\u001b[33m there\u001b[0m\u001b[33m was\u001b[0m\u001b[33m an\u001b[0m\u001b[33m issue\u001b[0m\u001b[33m with\u001b[0m\u001b[33m accessing\u001b[0m\u001b[33m the\u001b[0m\u001b[33m file\u001b[0m\u001b[33m.\u001b[0m\u001b[33m Since\u001b[0m\u001b[33m I\u001b[0m\u001b[33m don\u001b[0m\u001b[33m't\u001b[0m\u001b[33m have\u001b[0m\u001b[33m the\u001b[0m\u001b[33m ability\u001b[0m\u001b[33m to\u001b[0m\u001b[33m access\u001b[0m\u001b[33m or\u001b[0m\u001b[33m read\u001b[0m\u001b[33m files\u001b[0m\u001b[33m from\u001b[0m\u001b[33m your\u001b[0m\u001b[33m local\u001b[0m\u001b[33m system\u001b[0m\u001b[33m,\u001b[0m\u001b[33m I\u001b[0m\u001b[33m'll\u001b[0m\u001b[33m provide\u001b[0m\u001b[33m a\u001b[0m\u001b[33m general\u001b[0m\u001b[33m solution\u001b[0m\u001b[33m.\n", "\n", - "# Plot the average yearly inflation rate as a time series\n", - "plt.figure(figsize=(10, 6))\n", - "plt.plot(df['Year'], df['Yearly Inflation'], marker='o')\n", - "plt.title('Average Yearly Inflation Rate')\n", - "plt.xlabel('Year')\n", - "plt.ylabel('Inflation Rate (%)')\n", - "plt.grid(True)\n", - "plt.show()\n", - "tool_execution> Tool:code_interpreter Args:{'code': \"import pandas as pd\\nimport matplotlib.pyplot as plt\\n\\n# Read the CSV file\\ndf = pd.read_csv('/tmp/tmpco0s0o4_/LOdZoVp1inflation.csv')\\n\\n# Extract the year and inflation rate from the CSV file\\ndf['Year'] = pd.to_datetime(df['Year'], format='%Y')\\ndf = df.rename(columns={'Jan': 'Jan Rate', 'Feb': 'Feb Rate', 'Mar': 'Mar Rate', 'Apr': 'Apr Rate', 'May': 'May Rate', 'Jun': 'Jun Rate', 'Jul': 'Jul Rate', 'Aug': 'Aug Rate', 'Sep': 'Sep Rate', 'Oct': 'Oct Rate', 'Nov': 'Nov Rate', 'Dec': 'Dec Rate'})\\n\\n# Calculate the average yearly inflation rate\\ndf['Yearly Inflation'] = df[['Jan Rate', 'Feb Rate', 'Mar Rate', 'Apr Rate', 'May Rate', 'Jun Rate', 'Jul Rate', 'Aug Rate', 'Sep Rate', 'Oct Rate', 'Nov Rate', 'Dec Rate']].mean(axis=1)\\n\\n# Plot the average yearly inflation rate as a time series\\nplt.figure(figsize=(10, 6))\\nplt.plot(df['Year'], df['Yearly Inflation'], marker='o')\\nplt.title('Average Yearly Inflation Rate')\\nplt.xlabel('Year')\\nplt.ylabel('Inflation Rate (%)')\\nplt.grid(True)\\nplt.show()\"}\n", - "tool_execution> Tool:code_interpreter Response:completed\n", - "shield_call> No Violation\n", - "inference> This code reads the CSV file, extracts the year and inflation rate, calculates the average yearly inflation rate, and plots the average yearly inflation rate as a time series. The resulting plot shows the average inflation rate over the years.\n" + "\u001b[0m\u001b[33mTo\u001b[0m\u001b[33m plot\u001b[0m\u001b[33m the\u001b[0m\u001b[33m average\u001b[0m\u001b[33m yearly\u001b[0m\u001b[33m inflation\u001b[0m\u001b[33m as\u001b[0m\u001b[33m a\u001b[0m\u001b[33m time\u001b[0m\u001b[33m series\u001b[0m\u001b[33m,\u001b[0m\u001b[33m you\u001b[0m\u001b[33m'll\u001b[0m\u001b[33m need\u001b[0m\u001b[33m to\u001b[0m\u001b[33m have\u001b[0m\u001b[33m the\u001b[0m\u001b[33m following\u001b[0m\u001b[33m data\u001b[0m\u001b[33m:\n", + "\n", + "\u001b[0m\u001b[33m-\u001b[0m\u001b[33m A\u001b[0m\u001b[33m column\u001b[0m\u001b[33m with\u001b[0m\u001b[33m the\u001b[0m\u001b[33m year\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33m-\u001b[0m\u001b[33m A\u001b[0m\u001b[33m column\u001b[0m\u001b[33m with\u001b[0m\u001b[33m the\u001b[0m\u001b[33m inflation\u001b[0m\u001b[33m rate\u001b[0m\u001b[33m for\u001b[0m\u001b[33m each\u001b[0m\u001b[33m year\u001b[0m\u001b[33m\n", + "\n", + "\u001b[0m\u001b[33mHere\u001b[0m\u001b[33m's\u001b[0m\u001b[33m a\u001b[0m\u001b[33m general\u001b[0m\u001b[33m solution\u001b[0m\u001b[33m:\n", + "\n", + "\u001b[0m\u001b[33m1\u001b[0m\u001b[33m.\u001b[0m\u001b[33m Load\u001b[0m\u001b[33m the\u001b[0m\u001b[33m data\u001b[0m\u001b[33m into\u001b[0m\u001b[33m a\u001b[0m\u001b[33m pandas\u001b[0m\u001b[33m DataFrame\u001b[0m\u001b[33m.\n", + "\u001b[0m\u001b[33m2\u001b[0m\u001b[33m.\u001b[0m\u001b[33m Convert\u001b[0m\u001b[33m the\u001b[0m\u001b[33m '\u001b[0m\u001b[33myear\u001b[0m\u001b[33m'\u001b[0m\u001b[33m column\u001b[0m\u001b[33m to\u001b[0m\u001b[33m datetime\u001b[0m\u001b[33m.\n", + "\u001b[0m\u001b[33m3\u001b[0m\u001b[33m.\u001b[0m\u001b[33m Group\u001b[0m\u001b[33m by\u001b[0m\u001b[33m year\u001b[0m\u001b[33m and\u001b[0m\u001b[33m calculate\u001b[0m\u001b[33m the\u001b[0m\u001b[33m average\u001b[0m\u001b[33m inflation\u001b[0m\u001b[33m.\n", + "\u001b[0m\u001b[33m4\u001b[0m\u001b[33m.\u001b[0m\u001b[33m Plot\u001b[0m\u001b[33m the\u001b[0m\u001b[33m average\u001b[0m\u001b[33m yearly\u001b[0m\u001b[33m inflation\u001b[0m\u001b[33m as\u001b[0m\u001b[33m a\u001b[0m\u001b[33m time\u001b[0m\u001b[33m series\u001b[0m\u001b[33m using\u001b[0m\u001b[33m matplotlib\u001b[0m\u001b[33m.\n", + "\n", + "\u001b[0m\u001b[33mIf\u001b[0m\u001b[33m you\u001b[0m\u001b[33m provide\u001b[0m\u001b[33m the\u001b[0m\u001b[33m contents\u001b[0m\u001b[33m of\u001b[0m\u001b[33m the\u001b[0m\u001b[33m CSV\u001b[0m\u001b[33m file\u001b[0m\u001b[33m,\u001b[0m\u001b[33m I\u001b[0m\u001b[33m can\u001b[0m\u001b[33m help\u001b[0m\u001b[33m you\u001b[0m\u001b[33m plot\u001b[0m\u001b[33m the\u001b[0m\u001b[33m average\u001b[0m\u001b[33m yearly\u001b[0m\u001b[33m inflation\u001b[0m\u001b[33m as\u001b[0m\u001b[33m a\u001b[0m\u001b[33m time\u001b[0m\u001b[33m series\u001b[0m\u001b[33m.\u001b[0m\u001b[97m\u001b[0m\n", + "\u001b[30m\u001b[0m" ] } ], "source": [ + "from llama_stack_client.types.agents.turn_create_params import Document\n", + "\n", "agent_config = AgentConfig(\n", - " model=model_id,\n", + " sampling_params = {\n", + " \"max_tokens\" : 4096,\n", + " \"temperature\": 0.0\n", + " },\n", + " model=\"meta-llama/Llama-3.1-8B-Instruct\",\n", " instructions=\"You are a helpful assistant\",\n", - " tools=[\n", - " search_tool,\n", - " {\n", - " \"type\": \"code_interpreter\",\n", - " }\n", + " toolgroups=[\n", + " \"builtin::code_interpreter\",\n", + " \"builtin::websearch\"\n", " ],\n", - " tool_choice=\"required\",\n", + " tool_choice=\"auto\",\n", " input_shields=[],\n", " output_shields=[],\n", " enable_session_persistence=False,\n", ")\n", - "\n", "codex_agent = Agent(client, agent_config)\n", "session_id = codex_agent.create_session(\"test-session\")\n", "\n", - "user_prompts = [\n", - " (\n", - " \"Here is a csv, can you describe it ?\",\n", - " [\n", - " Attachment(\n", - " content=\"https://raw.githubusercontent.com/meta-llama/llama-stack-apps/main/examples/resources/inflation.csv\",\n", - " mime_type=\"test/csv\",\n", - " )\n", - " ],\n", - " ),\n", - " (\"Which year ended with the highest inflation ?\", None),\n", - " (\n", - " \"What macro economic situations that led to such high inflation in that period?\",\n", - " None,\n", - " ),\n", - " (\"Plot average yearly inflation as a time series\", None),\n", + "\n", + "inflation_doc = Document(\n", + " content=\"https://raw.githubusercontent.com/meta-llama/llama-stack-apps/main/examples/resources/inflation.csv\",\n", + " mime_type=\"text/csv\",\n", + ")\n", + "\n", + "user_input = [\n", + " {\"prompt\": \"Here is a csv, can you describe it?\", \"documents\": [inflation_doc]},\n", + " {\"prompt\": \"Plot average yearly inflation as a time series\"},\n", "]\n", "\n", - "for prompt in user_prompts:\n", - " cprint(f'User> {prompt}', 'green')\n", + "for input in user_input:\n", + " cprint(f'User> {input[\"prompt\"]}', 'green')\n", " response = codex_agent.create_turn(\n", + "\n", " messages=[\n", " {\n", " \"role\": \"user\",\n", - " \"content\": prompt[0],\n", + " \"content\": input[\"prompt\"],\n", " }\n", " ],\n", - " attachments=prompt[1],\n", " session_id=session_id,\n", + " documents=input.get(\"documents\", None)\n", " )\n", " # for chunk in response:\n", " # print(chunk)\n", @@ -1826,12 +2319,12 @@ "height": 564 }, "id": "JqBBVLKdIHHq", - "outputId": "4563e803-8385-426b-ec6c-e8b19e2ee6e6" + "outputId": "3c89c303-e7c0-4ae2-c271-f34a4d296a85" }, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0EAAAIjCAYAAADFthA8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB+WklEQVR4nO3dd3hUZdrH8d+k90BCGiSE0AkBpFdFVJoUscGiKCq6rmt3XffVVQFdd3Vd265tbdjAguIKKiACgvReQi+hh4QQSCGkzZz3j5BITIBkmJkzyXw/15ULcubknPvcmYG553nO/VgMwzAEAAAAAB7Cy+wAAAAAAMCVKIIAAAAAeBSKIAAAAAAehSIIAAAAgEehCAIAAADgUSiCAAAAAHgUiiAAAAAAHoUiCAAAAIBHoQgCAAAA4FEoggAAbu3yyy/X5ZdfbnYYFT755BO1bdtWvr6+atCggSTnxDhp0iRZLBaHHhMAUIYiCIDHevPNN2WxWNSzZ0+zQ3Eby5cvl5eXlx5//PFqH3/hhRdksVj0/fffuzgyx7FYLLrvvvvs+tnt27frtttuU4sWLfTuu+/qnXfeuahYCgoKNGnSJP38888XdRxHs1gslb7CwsLUv3//i/q9T5s2Ta+++qrjggSAi0ARBMBjTZ06Vc2aNdOqVau0e/dus8NxC71799bdd9+tl156SVu2bKn02P79+/XMM8/oxhtv1LBhw0yK0Fw///yzbDabXnvtNd12220aPXr0RR2voKBAkydPrrYIevLJJ3X69OmLOv7FGDhwoD755BN9/PHHeuyxx7R7926NGDFCc+fOtet4FEEA3AlFEACPlJaWpmXLlunll19WVFSUpk6d6vIYbDabCgsLXX7eC3n++efVqFEj3X333TIMo2L7/fffL19fX7322msuiaOgoMAl56mNzMxMSaqYBudMPj4+CggIcPp5zqV169YaN26cbrnlFj355JP66aefZBiGy37/AOBMFEEAPNLUqVPVsGFDDRs2TDfccEOlIqikpEQRERG6/fbbq/xcbm6uAgIC9Oijj1ZsKyoq0sSJE9WyZUv5+/srISFBjz32mIqKiir9bPk0rKlTp6p9+/by9/fXnDlzJEn/+te/1KdPH0VGRiowMFBdu3bVV199VeX8p0+f1gMPPKBGjRopNDRUI0eO1OHDh2WxWDRp0qRK+x4+fFh33HGHYmJi5O/vr/bt2+uDDz64YG7Cw8P12muvaenSpXrvvfckSd98841mzZql559/XnFxcbLZbHr11VfVvn17BQQEKCYmRnfffbdOnDhR6Vjffvuthg0bpsaNG8vf318tWrTQs88+K6vVWmm/yy+/XCkpKVq7dq0uu+wyBQUF6YknnqgSW35+voKDg/Xggw9WeezQoUPy9vbWP/7xjwte49l+/vlnWSwWffnll3ruuecUHx+vgIAAXXnllZVGCJs1a6aJEydKkqKioqrNebni4mI9/fTT6tq1q8LDwxUcHKxLL71UCxcurNhn3759ioqKkiRNnjy5YupZ+TGruyeotLRUzz77rFq0aCF/f381a9ZMTzzxRJXnWrNmzTR8+HAtWbJEPXr0UEBAgJo3b66PP/64Vrk5W7t27dSoUSPt2bOn0vaa/I4vv/xyff/999q/f3/FdTZr1qzi8Zq+hgDAYQwA8EBt27Y1JkyYYBiGYSxevNiQZKxatari8TvuuMNo0KCBUVRUVOnnPvroI0OSsXr1asMwDMNqtRqDBg0ygoKCjIceesj473//a9x3332Gj4+Pcc0111T6WUlGu3btjKioKGPy5MnGG2+8Yaxfv94wDMOIj483/vjHPxqvv/668fLLLxs9evQwJBnfffddpWOMHj3akGTccsstxhtvvGGMHj3a6NSpkyHJmDhxYsV+R48eNeLj442EhATjmWeeMd566y1j5MiRhiTjlVdeqVGOhg0bZjRs2NDYs2ePkZCQYPTp08ew2WyGYRjGnXfeafj4+Bh33XWX8fbbbxt/+ctfjODgYKN79+5GcXFxxTFGjRpljB492njxxReNt956y7jxxhsNScajjz5a6Vz9+/c3YmNjjaioKOP+++83/vvf/xr/+9//Kh7r379/xb4333yzERMTY5SWllY6xj//+U/DYrEY+/fvP+91STLuvffeiu8XLlxoSDI6d+5sdO3a1XjllVeMSZMmGUFBQUaPHj0q9vvmm2+Ma6+91pBkvPXWW8Ynn3xibNy4sdoYjx07ZsTFxRmPPPKI8dZbbxn//Oc/jTZt2hi+vr4Vv/P8/HzjrbfeMiQZ1157rfHJJ59UOubEiRON3/43PX78eEOSccMNNxhvvPGGceuttxqSjFGjRlXaLzEx0WjTpo0RExNjPPHEE8brr79udOnSxbBYLEZqaup581NdjgzDME6ePGl4e3sbPXv2rLS9Jr/jH3/80bjkkkuMRo0aVVznN998YxhG7V5DAOAoFEEAPM6aNWsMSca8efMMwzAMm81mxMfHGw8++GDFPnPnzjUkGbNmzar0s1dffbXRvHnziu8/+eQTw8vLy/jll18q7ff2228bkoylS5dWbJNkeHl5GVu2bKkSU0FBQaXvi4uLjZSUFOOKK66o2LZ27VpDkvHQQw9V2ve2226rUgRNmDDBiIuLM7Kysirt+7vf/c4IDw+vcr7q7Nu3zwgODjYiIiIMX19fY/PmzYZhGMYvv/xiSDKmTp1aaf85c+ZU2V7dee6++24jKCjIKCwsrNjWv39/Q5Lx9ttvV9n/twVG+e9m9uzZlfbr2LFjpf3O5VxFULt27SoVva+99pohqeK6DePXwuTYsWPnjbG0tLRKAX3ixAkjJibGuOOOOyq2HTt2rMrv7rfnKrdhwwZDknHnnXdW2u/RRx81JBkLFiyo2JaYmGhIMhYvXlyxLTMz0/D39zf+9Kc/nSs1FSQZEyZMMI4dO2ZkZmYaa9asMYYMGWJIMl588cVK+9b0dzxs2DAjMTGxyr61eQ0BgKMwHQ6Ax5k6dapiYmI0YMAASWXT1MaMGaPPP/+8YgrPFVdcoUaNGumLL76o+LkTJ05o3rx5GjNmTMW26dOnq127dmrbtq2ysrIqvq644gpJqjT9SZL69++v5OTkKjEFBgZWOk9OTo4uvfRSrVu3rmJ7+dS5P/7xj5V+9v7776/0vWEY+vrrrzVixAgZhlEprsGDBysnJ6fScc8lMTFREydOVHZ2th555BGlpKRUXHN4eLgGDhxY6dhdu3ZVSEhIpWs++7ry8vKUlZWlSy+9VAUFBdq+fXul8/n7+1c7BfG3rrrqKjVu3LjSFMbU1FRt2rRJ48aNu+DPn8vtt98uPz+/iu8vvfRSSdLevXtrfSxvb++KY9lsNmVnZ6u0tFTdunWrUe6r88MPP0iSHnnkkUrb//SnP0lSlc5tycnJFdcglU3ha9OmTY2v5/3331dUVJSio6PVrVs3zZ8/X4899liV89fmd1yd2r6GAMARfMwOAABcyWq16vPPP9eAAQOUlpZWsb1nz5566aWXNH/+fA0aNEg+Pj66/vrrNW3aNBUVFcnf318zZsxQSUlJpSJo165d2rZtW8W9Hb9VfiN9uaSkpGr3++677/S3v/1NGzZsqHQfxNn3hOzfv19eXl5VjtGyZctK3x87dkwnT57UO++8c84Wzr+N61y6d+8uSerWrVvFtl27diknJ0fR0dEXPPaWLVv05JNPasGCBcrNza20X05OTqXvmzRpUqkIORcvLy/dfPPNeuutt1RQUKCgoCBNnTpVAQEBuvHGG2t0XdVp2rRppe8bNmwoSVXuc6qpjz76SC+99JK2b9+ukpKSiu3neg5cSPnv/7e/79jYWDVo0ED79++vtP231yOVXVNNr+eaa67Rfffdp+LiYq1evVp///vfVVBQIC+vyp+f1uZ3XJ3avoYAwBEoggB4lAULFig9PV2ff/65Pv/88yqPT506VYMGDZIk/e53v9N///tfzZ49W6NGjdKXX36ptm3bqlOnThX722w2dejQQS+//HK150tISKj0/dmfmpf75ZdfNHLkSF122WV68803FRcXJ19fX02ZMkXTpk2r9TXabDZJ0rhx4zR+/Phq9+nYsWOtj3v28aOjo8/ZUa/8zezJkyfVv39/hYWF6ZlnnlGLFi0UEBCgdevW6S9/+UtFnOWqy8253HrrrXrxxRf1v//9T2PHjtW0adM0fPhwhYeH231d3t7e1W43zuqQV1OffvqpbrvtNo0aNUp//vOfFR0dXdG04beNBWqrpguoXuz1xMfH66qrrpIkXX311WrUqJHuu+8+DRgwQNddd52k2v+Oq1Pb1xAAOAJFEACPMnXqVEVHR+uNN96o8tiMGTP0zTff6O2331ZgYKAuu+wyxcXF6YsvvlC/fv20YMEC/fWvf630My1atNDGjRt15ZVX1vjN6W99/fXXCggI0Ny5c+Xv71+xfcqUKZX2S0xMlM1mU1pamlq1alWx/bdrHEVFRSk0NFRWq7XiTawjtWjRQj/99JP69u173sLl559/1vHjxzVjxgxddtllFdvPHoGzV0pKijp37qypU6cqPj5eBw4c0H/+85+LPq6jfPXVV2revLlmzJhR6XlR3l2uXG2eM+W//127dqldu3YV2zMyMnTy5EklJiZefODncffdd+uVV17Rk08+qWuvvVYWi6VWv+NzXasjXkMAUFvcEwTAY5w+fVozZszQ8OHDdcMNN1T5uu+++5SXl6eZM2dKKpt2dcMNN2jWrFn65JNPVFpaWmkqnCSNHj1ahw8f1rvvvlvt+U6dOnXBuLy9vWWxWCq1FN63b5/+97//Vdpv8ODBkqQ333yz0vbfvvn39vbW9ddfr6+//lqpqalVznfs2LELxnQ+o0ePltVq1bPPPlvlsdLSUp08ebIiDqnyyENxcXGV+O11yy236Mcff9Srr76qyMhIDR061CHHdYTqrn3lypVavnx5pf2CgoIkqSJn53P11VdLUpUFR8tHUJy9gK2Pj4/+9Kc/adu2bfr2228l1e53HBwcXO30OEe8hgCgthgJAuAxZs6cqby8PI0cObLax3v16lWxcGp5sTNmzBj95z//0cSJE9WhQ4dKn8BLZW/Ev/zyS/3hD3/QwoUL1bdvX1mtVm3fvl1ffvml5s6dW+l+muoMGzZML7/8soYMGaKbbrpJmZmZeuONN9SyZUtt2rSpYr+uXbvq+uuv16uvvqrjx4+rV69eWrRokXbu3Cmp8iftzz//vBYuXKiePXvqrrvuUnJysrKzs7Vu3Tr99NNPys7OtiuHUllzh7vvvlv/+Mc/tGHDBg0aNEi+vr7atWuXpk+frtdee0033HCD+vTpo4YNG2r8+PF64IEHZLFY9Mknn9g1vaw6N910kx577DF98803uueee+Tr6+uQ4zrC8OHDNWPGDF177bUaNmyY0tLS9Pbbbys5OVn5+fkV+wUGBio5OVlffPGFWrdurYiICKWkpFQ0oThbp06dNH78eL3zzjsV09BWrVqljz76SKNGjapo9OFMt912m55++mm98MILGjVqVK1+x127dtUXX3yhRx55RN27d1dISIhGjBjhkNcQANSaaX3pAMDFRowYYQQEBBinTp065z633Xab4evrW9Fa2mazGQkJCYYk429/+1u1P1NcXGy88MILRvv27Q1/f3+jYcOGRteuXY3JkycbOTk5FfupmrVXyr3//vtGq1atDH9/f6Nt27bGlClTql0n5tSpU8a9995rREREGCEhIcaoUaOMHTt2GJKM559/vtK+GRkZxr333mskJCQYvr6+RmxsrHHllVca77zzTo3yZRi/to+ePn16lcfeeecdo2vXrkZgYKARGhpqdOjQwXjssceMI0eOVOyzdOlSo1evXkZgYKDRuHFj47HHHqtocb1w4cKK/fr372+0b9++2hh+2376bFdffbUhyVi2bFmNr+m3v4dzXWNaWpohyZgyZUrFtpq2yLbZbMbf//53IzEx0fD39zc6d+5sfPfdd8b48eOrtIletmyZ0bVrV8PPz69Su+zqfv8lJSXG5MmTjaSkJMPX19dISEgwHn/88UqtqA2jrEX2sGHDqlz7+XJ5tvM9VydNmlTp91fT33F+fr5x0003GQ0aNDAkVcpDTV9DAOAoFsNw0EdyAABTbNiwQZ07d9ann36qm2++2exwXOraa6/V5s2bq9wXBQDA+XBPEADUIadPn66y7dVXX5WXl1elG9M9QXp6ur7//nvdcsstZocCAKhjuCcIAOqQf/7zn1q7dq0GDBggHx8fzZ49W7Nnz9bvf/97j2klnJaWpqVLl+q9996Tr6+v7r77brNDAgDUMRRBAFCH9OnTR/PmzdOzzz6r/Px8NW3aVJMmTarSurs+W7RokW6//XY1bdpUH330kWJjY80OCQBQx3BPEAAAAACPwj1BAAAAADwKRRAAAAAAj1Kn7wmy2Ww6cuSIQkNDKy0SCAAAAMCzGIahvLw8NW7cWF5e5x/rqdNF0JEjRzymGxIAAACACzt48KDi4+PPu0+dLoJCQ0MllV1oWFiYqbGUlJToxx9/1KBBg+Tr62tqLHUNubMPebMPebMfubMPebMPebMPebMfubOPO+UtNzdXCQkJFTXC+dTpIqh8ClxYWJhbFEFBQUEKCwsz/QlQ15A7+5A3+5A3+5E7+5A3+5A3+5A3+5E7+7hj3mpymwyNEQAAAAB4FIogAAAAAB6FIggAAACAR6EIAgAAAOBRKIIAAAAAeBSKIAAAAAAehSIIAAAAgEehCAIAAADgUSiCAAAAAHgUiiAAAAAAHoUiCAAAAIBHoQgCAAAA4FEoggAAAAB4FIogAAAAeDSrzdDKtGytzbJoZVq2rDbD7JDgZD5mBwAAAACYZU5quibP2qr0nEJJ3vp41xrFhQdo4ohkDUmJMzs8OAkjQQAAAPBIc1LTdc+n684UQL86mlOoez5dpzmp6SZFBmejCAIAAIDHsdoMTZ61VdVNfCvfNnnWVqbG1VMUQQAAAPA4q9Kyq4wAnc2QlJ5TqFVp2a4LCi5DEQQAAACPk5l37gLInv1Qt1AEAQAAwONEhwY4dD/ULRRBAAAA8Dg9kiIUF37uAsciKS48QD2SIlwXFFyGIggAAAAex9vLookjks/5uCFp4ohkeXtZXBcUXIYiCAAAAB7pynYxCvLzrvaxZpFBGpQc6+KI4CoUQQAAAPBIK/dmq6DYqoggX310W1fd2sqqf4/pqCBfL+07XqDpaw+aHSKchCIIAAAAHmn2mcVQB6fEqk+LSHVtZGhoSqweGdRGkvT87O06carYzBDhJBRBAAAA8DhWm6G5WzIkSYPbV572Nr5PM7WJCdWJghK9+OMOM8KDk1EEAQAAwOOsP3BCWflFCg3wUZ8WjSo95uvtpWeuaS9J+mzVAW08eNKECOFMFEEAAADwOLNTj0qSrmoXIz+fqm+JezaP1LWdm8gwpKe+TZXVZrg6RDiR6UXQ4cOHNW7cOEVGRiowMFAdOnTQmjVrzA4LAAAA9ZRhGJpzpgj67VS4sz1+dVuF+vto06Ecfb76gKvCgwuYWgSdOHFCffv2la+vr2bPnq2tW7fqpZdeUsOGDc0MCwAAAPVY6uFcHT55WoG+3urfOuqc+0WHBuiRQa0lSf+cs0PZNEmoN3zMPPkLL7yghIQETZkypWJbUlKSiREBAACgvpuzpawr3OVtohR4jnWCyt3SK1Ffrjmkbem5emH2dr1wQ0dXhAgnM7UImjlzpgYPHqwbb7xRixYtUpMmTfTHP/5Rd911V7X7FxUVqaioqOL73NxcSVJJSYlKSkpcEvO5lJ/f7DjqInJnH/JmH/JmP3JnH/JmH/JmH/JWM7M3l02FG9guqkrOqsvdxGFt9Lv3VuuLNQd1fZc4dU5o4LJY3Z07PedqE4PFMAzT7vIKCAiQJD3yyCO68cYbtXr1aj344IN6++23NX78+Cr7T5o0SZMnT66yfdq0aQoKCnJ6vAAAAKjbjhZI/9joI2+Lob93syqghkMCU3d7adUxL8UHG/pTB6u8LM6NE7VXUFCgm266STk5OQoLCzvvvqYWQX5+furWrZuWLVtWse2BBx7Q6tWrtXz58ir7VzcSlJCQoKysrAteqLOVlJRo3rx5GjhwoHx9fU2Npa4hd/Yhb/Yhb/Yjd/Yhb/Yhb/Yhbxf2xs979er83bq8dSO9e0uXiu0Xyt3x/CINem2pcgtLNXF4W43r2dSVYbstd3rO5ebmqlGjRjUqgkydDhcXF6fk5ORK29q1a6evv/662v39/f3l7+9fZbuvr6/pSS/nTrHUNeTOPuTNPuTNfuTOPuTNPuTNPuTt3H7cmilJurpD42pzdK7cxTb01Z8Ht9FT327Ryz/t1ohL4tUopOr7Uk/lDs+52pzf1O5wffv21Y4dlVfh3blzpxITE02KCAAAAPXVgeMF2pqeK28vi65Kjqn1z9/UM1HtG4cpr7BUz8/e7oQI4SqmFkEPP/ywVqxYob///e/avXu3pk2bpnfeeUf33nuvmWEBAACgHirvCtczKUIRwX61/nlvL4ueHZUiSfpq7SGt2Zft0PjgOqYWQd27d9c333yjzz77TCkpKXr22Wf16quv6uabbzYzLAAAANRD5QukDkk59wKpF9KlaUP9rnuCJOnJ/6Wq1GpzSGxwLVPvCZKk4cOHa/jw4WaHAQAAgHosI7dQ6w6clCQNbm9/ESRJjw1pq9mpR7X9aJ4+WbFft/dlncu6xtSRIAAAAMAV5m4pGwXq0rSBYsICLupYEcF+emxIG0nSyz/uVGZu4UXHB9eiCAIAAEC954ipcGf7Xfem6hQfrryiUv2DJgl1DkUQAAAA6rXsU8VamVbWxGBI+ziHHNPby6JnrkmRxSJ9s/6wVu497pDjwjUoggAAAFCv/bQ1Q1aboeS4MDWNDHLYcTslNNDYHmWLpj71bapKaJJQZ1AEAQAAoF6bc+Z+oKEOmgp3tscGt1HDIF/tzMjXR8v2Ofz4cA6KIAAAANRbeYUlWrIrS5Lj7gc6W4MgP/3f0LaSpFfm7VQGTRLqBIogAAAA1FsLtmeq2GpT86hgtYwOcco5buyaoM5NG+hUsVV/+36bU84Bx6IIAgAAQL1V3hVuaEqsLBaLU87h5WXRs9ekyMsizdp4RMt2ZznlPHAciiAAAADUS6eLrfp5xzFJjusKdy4pTcI1rleiJOnpmVtUXEqTBHdGEQQAAIB6afGuYzpdYlWTBoFKaRLm9PP9aWAbRQb7aXdmvj5Ymub088F+FEEAAACol85eINVZU+HOFh7kq8evbidJ+vf8XTpy8rTTzwn7UAQBAACg3ikutemnbRmSnNMa+1yu69xE3RIbqqDYqudokuC2KIIAAABQ7yzbk6W8wlJFhfqrS9OGLjuvl5dFz5xpkvD95nQt3nnMZedGzVEEAQAAoN6Ze2aB1EHJMfLycv5UuLMlNw7T+D7NJEmTZm5RUanVpefHhVEEAQAAoF6x2gz9uKV8Kpxzu8Kdy8MDW6tRiL/2Zp3Se7/QJMHdUAQBAACgXlm9L1vHTxUrPNBXPZtHmBJDWICv/jqsrSTpPwt26dCJAlPiQPUoggAAAFCvlHeFG5gcI19v897ujrqkiXokRaiwxKZnv9tqWhyoiiIIAAAA9YbNZlTcDzSkveu6wlXHYrHo2WtS5O1l0dwtGVq4I9PUePAriiAAAADUG5sO5yg9p1DBft7q16qR2eGoTWyobj+rSUJhCU0S3AFFEAAAAOqN2anpkqQBbaMV4OttcjRlHhrYWjFh/tp/vEDvLN5rdjgQRRAAAADqCcMwNPfM/UBDXLhA6oWE+Pvor8OSJUlvLNytg9k0STAbRRAAAADqhR0Zedp3vEB+Pl4a0Cba7HAqGdExTr2bR6qo1KbJs7aYHY7HowgCAABAvTB7c9ko0GWtohTs72NyNJVZLBY9O6q9fLws+mlbpn7ammF2SB6NIggAAAD1QkVXODeaCne2ltGhmnBpkiRp8nc0STATRRAAAADqvLSsU9p+NE8+XhZd1c69psKd7YErWikuPEAHs0/rzZ/3mB2Ox6IIAgAAQJ1XvkBq7xaRahDkZ3I05xbs76Onhpc1SXh70R7tyzplckSeiSIIAAAAdd4cN58Kd7ahKbG6tFUjFZfaNGnWFhmGYXZIHociCAAAAHXakZOntfHgSVks0sDkGLPDuSCLxaJJI9vL19uin3cc0480SXA5iiAAAADUaeUNEbonRig6NMDkaGqmRVSIfn9Zc0nSM7O26nQxTRJciSIIAAAAddrsM/cDDa4DU+HOdu+AlmrSIFCHT57WGwt3mx2OR6EIAgAAQJ11LK9Iq/dlS5IGt3f/qXBnC/L7tUnCO4v3au+xfJMj8hwUQQAAAKizftqWIcOQOsaHK75hkNnh1Nrg9jG6vE2Uiq02TZxJkwRXoQgCAABAnVUxFa593ZoKV85isWjSiPby8/bSL7uyKlp9w7koggAAAFAn5Zwu0bLdWZLK2k7XVc0aBesP/c80Sfhuq04VlZocUf1HEQQAAIA6af62DJXaDLWOCVHzqBCzw7kofxzQUvENA5WeU6j/LKBJgrNRBAEAAKBOKp86NqSOToU7W4CvtyaNaC9Jeu+XvdqdmWdyRPUbRRAAAADqnFNFpVq085gkaUhKnMnROMZVyTG6sm20Sm2Gnv6WJgnORBEEAACAOmfRzmMqKrWpaUSQ2sWFmh2Ow0wa2V7+Pl5atue4vtuUbnY49RZFEAAAAOqc8qlwQ1NiZbFYTI7GcRIigvTHy1tKkv72/Vbl0yTBKSiCAAAAUKcUlVq1YHumJGlwHe4Kdy5392+uxMggZeQW6bWfdpodTr1EEQQAAIA6ZenuLOUXlSomzF+XxDcwOxyHC/D11qSRZU0SPli6TzuO0iTB0SiCAAAAUKfM3vxrVzgvr/ozFe5sA9pEa1ByjKw2Q09/m0qTBAejCAIAAECdUWq1ad62DEn1cyrc2Z4ekawAXy+tTMvWtxuOmB1OvUIRBAAAgDpjVVq2ThaUKCLYTz2aRZgdjlPFNwzS/Ve0kiQ998M25RaWmBxR/UERBAAAgDpj9pmucAPbxcjHu/6/lb3z0iQlNQrWsbwivTpvl9nh1Bv1/5kDAACAesFmMzR3y5n7gTrU76lw5fx9vDX5TJOEj5bv07b0XJMjqh8oggAAAFAnrD94Qpl5RQr191GfFpFmh+Myl7WO0tUdYmW1GXrqfzRJcASKIAAAANQJ5QukXtEuWv4+3iZH41pPDktWoK+31uw/oRnrDpsdTp1HEQQAAAC3ZxiG5pyZCje0nneFq07jBoF64MqyJgn/mL1NOadpknAxKIIAAADg9rYcydXB7NMK8PXSZa2jzA7HFBP6JalFVLCy8ov18o87zA6nTqMIAgAAgNsrb4hweetoBfn5mByNOfx8vPTMNSmSpE9W7Ffq4RyTI6q7KIIAAADg9spbYw/xwKlwZ+vbspGGd4yTzZCe+jZVNhtNEuxBEQQAAAC3tjszT7sz8+XrbdGAttFmh2O6J4clK9jPW+sPnNRXaw+ZHU6dRBEEAAAAtzZ3S4akslGQ8EBfk6MxX2x4gB66qrUk6fk523WyoNjkiOoeiiAAAAC4tdmp6ZKkIe09eyrc2W7r20ytY0KUfapYL86lSUJtUQQBAADAbR3MLlDq4Vx5WaSByTFmh+M2fL1/bZIwbdUBbTp00tyA6hiKIAAAALit8q5wPZIiFBnib3I07qVX80iNuqSxDEN66n80SagNiiAAAAC4rTnlXeGYCletJ65up1B/H208lKPPVx80O5w6gyIIAAAAbikzt1BrD5yQJA328NbY5xIdFqCHB5Y1Sfjn3O3KPkWThJqgCAIAAIBbmrs1Q4YhXZLQQHHhgWaH47Zu7Z2otrGhOllQohfnbjc7nDqBIggAAABuae6ZqXBDGQU6Lx9vLz07qqxJwuerD2r9mdEznBtFEAAAANzOiVPFWr73uCRpCEXQBXVvFqHru8SXNUn4NlVWmiScF0UQAAAA3M5P2zJktRlqFxemxMhgs8OpE/5vaFuFBvgo9XCupq06YHY4bo0iCAAAAG6HrnC1FxXqr0cHtZEkvThnu7Lyi0yOyH1RBAEAAMCt5BeV6pddWZKYCldb43olqn3jMOUWluqF2TRJOBeKIAAAALiVhdszVWy1qXmjYLWOCTE7nDrF28uiZ64pa5Iwfe0hrd2fbXJE7okiCAAAAG6lfCrc4JRYWSwWk6Ope7omNtTobvGSpCf/t0WlVpvJEbkfiiAAAAC4jcISqxbuyJREa+yL8ZchbRUe6Ktt6bn6dMV+s8NxOxRBAAAAcBuLdx5TQbFVjcMD1KFJuNnh1FmRIf768+CyJgkv/bhTx/JoknA2iiAAAAC4jTlbmArnKGN7NFXH+HDlFZXqHz9sMzsct0IRBAAAALdQYrXpp60ZkqShKXEmR1P3eXtZ9Ow1KbJYpBnrD2vlmcVnQREEAAAAN7F8z3HlFpaqUYifuiY2NDuceqFTQgP9rntTSdLT325RCU0SJFEEAQAAwE2UT4Ub1D5W3l5MhXOUxwa3UcMgX+3IyNNHy/aZHY5boAgCAACA6aw2Qz+eKYKGtKcrnCM1DPbTX4a0lSS9+tMuZeQWmhyR+SiCAAAAYLq1+08oK79YYQE+6tU80uxw6p3R3RLUKaGB8otK9XeaJFAEAQAAwHyzU9MlSVclx8jPh7eojublZdHfzjRJ+HbDES3bk2V2SKbiGQYAAABTGYahualMhXO2DvHhGtczURJNEiiCAAAAYKrNh3N0JKdQQX7euqx1lNnh1GuPDmqjiGA/7c7M15SlaWaHYxqKIAAAAJhq9plRoAFtohXg621yNPVbeJCv/m/or00S0nNOmxyROSiCAAAAYBrDMDSnfCpcClPhXOGGLvHqmthQBcVW/e17z2ySQBEEAAAA0+zMyFda1in5eXtpQNtos8PxCF5eFj1zTXt5WaTvN6VryS7Pa5JAEQQAAADTlI8CXdqqkUL8fUyOxnO0bxyuW3s3kyQ9PTNVxaWe1SSBIggAAACmmbOFqXBmeXhgazUK8dfeY6f03pK9ZofjUqYWQZMmTZLFYqn01bZtWzNDAgAAgIvsP35K29Jz5e1l0VXtYswOx+OEB/rqiavL3nv/Z/5uHT7pOU0STB8Jat++vdLT0yu+lixZYnZIAAAAcIHyqXC9m0eqYbCfydF4pms7N1GPZhE6XWLV377banY4LmN6EeTj46PY2NiKr0aNGpkdEgAAAFygvDX2YKbCmcZiseiZUe3l7WXR7NSjWrTzmNkhuYTpd5/t2rVLjRs3VkBAgHr37q1//OMfatq0abX7FhUVqaioqOL73NxcSVJJSYlKSkpcEu+5lJ/f7DjqInJnH/JmH/JmP3JnH/JmH/Jmn7qUt/ScQm04eFIWi3RF60jTY65LuXO0FpGBurVXU01Ztl9P/y9V39/fR/4+NRsrcae81SYGi2EYhhNjOa/Zs2crPz9fbdq0UXp6uiZPnqzDhw8rNTVVoaGhVfafNGmSJk+eXGX7tGnTFBQU5IqQAQAA4ACL0y36ep+3kkINPZRiNTscj1dYKj23wVu5JRYNS7BqULxpJYLdCgoKdNNNNyknJ0dhYWHn3dfUIui3Tp48qcTERL388suaMGFClcerGwlKSEhQVlbWBS/U2UpKSjRv3jwNHDhQvr6+psZS15A7+5A3+5A3+5E7+5A3+5A3+9SlvI37YLVWpp3Q40Na646+zcwOp07lzllmbUrXI9M3K8DXS7Pv76v4hoEX/Bl3yltubq4aNWpUoyLI9OlwZ2vQoIFat26t3bt3V/u4v7+//P39q2z39fU1Penl3CmWuobc2Ye82Ye82Y/c2Ye82Ye82cfd83Y8v0ir952QJF3dsYlbxeruuXOma7sk6Mu1h7Vib7b+Pmen3r21W41/1h3yVpvzm94Y4Wz5+fnas2eP4uLizA4FAAAATjJva4ZshpTSJEwJEdzS4C4sFouevSZFPl4WzduaoQXbM8wOyWlMLYIeffRRLVq0SPv27dOyZct07bXXytvbW2PHjjUzLAAAADhRxQKp7ekK525axYRqQr8kSdKkmVtVWFI/79cytQg6dOiQxo4dqzZt2mj06NGKjIzUihUrFBUVZWZYAAAAcJLcwhIt3Z0lSRqSwuwfd3T/la0UGxagA9kFenvRHrPDcQpT7wn6/PPPzTw9AAAAXGzBtkyVWA21jA5Ry+gQs8NBNUL8ffTk8Ha6b9p6vfnzHl3XOV5NI+vXtEW3uicIAAAA9ducMwukDmWBVLc2rEOc+rVspOJSmybN2iI3aijtEBRBAAAAcImC4lL9vDNTkjSY+4HcmsVi0aSR7eXrbdGC7Zn6aVum2SE5FEUQAAAAXGLxzmMqLLEpISJQ7Rubu8YjLqxldIjuvLS5JGnSzC06XVx/miRQBAEAAMAlZqf+2hXOYrGYHA1q4v4rWqpxeIAOnzytN3+ufi3PuogiCAAAAE5XVGrVgjNTqoZwP1CdEeTno6dHJEuS/rtor9KyTpkckWNQBAEAAMDplu05rryiUkWH+qtzQkOzw0EtDG4fq8taR6nYatPEmfWjSQJFEAAAAJxuzuayqXCD28fKy4upcHWJxWLR5JHt5eftpcU7j2numcVu6zKKIAAAADhVqdWmedsyJNEau65KahSsu/uXNUl4ZtZWFRSXmhzRxaEIAgAAgFOt2pet7FPFahDkqx5JEWaHAzv98fKWatIgUEdyCvX6grrdJIEiCAAAAE4190xXuIHtYuTjzdvPuirQz1uTRraXJL37y17tOJqnlWnZWptl0cq0bFltdedeIR+zAwAAAED9ZbMZmrvlzFS4DkyFq+uuahetK9pGa8H2TI34zxIVW22SvPXxrjWKCw/QxBHJGpISZ3aYF0QpDgAAAKfZcOikjuYWKsTfR31bNjI7HFwki8WiAW2iJOlMAfSrozmFuufTdZqTmm5GaLVCEQQAAACnKZ8Kd0XbaPn7eJscDS6W1WbozZ/3VPtY+WS4ybO2uv3UOIogAAAAOIVhGJp9pghigdT6YVVattJzCs/5uCEpPadQq9KyXReUHSiCAAAA4BTb0vN0ILtA/j5e6t86yuxw4ACZeecugOzZzywUQQAAAHCKOWcW1ezfOkrB/vTjqg+iQwMcup9ZKIIAAADgFOU3yDMVrv7okRShuPAAWc7xuEVSXHiA268HRREEAAAAh9tzLF87M/Ll42XRle1izA4HDuLtZdHEEcmSVKUQKv9+4ohkeXudq0xyDxRBAAAAcLg5Zxoi9GnZSOGBviZHA0cakhKnt8Z1UWx45SlvseEBemtclzqxThCTMwEAAOBwc8/cDzSUqXD10pCUOA1MjtXy3Zn68ZeVGnRpT/VuGe32I0DlKIIAAADgUIdOFGjToRxZLNLAZKbC1VfeXhb1TIrQ8W2GeiZF1JkCSGI6HAAAABxs7pYMSVL3ZhFqFOJvcjRAVRRBAAAAcKi5qUyFg3ujCAIAAIDDZOYVavX+bEnS4PYUQXBPFEEAAABwmHlbM2QYUqeEBmrcINDscIBqUQQBAADAYcpbYw9hFAhujCIIAAAADnGyoFjL9xyXJA3hfiC4MYogAAAAOMT8bZkqtRlqGxuqpEbBZocDnBNFEAAAABxi9pmpcDREgLujCAIAAMBFO1VUqsW7jkmShnagCIJ7owgCAADARVu4I1PFpTY1iwxSm5hQs8MBzosiCAAAABetvCvc4JRYWSwWk6MBzo8iCAAAABelsMSqhdszJUlDU+JMjga4MIogAAAAXJQlu7J0qtiquPAAdWwSbnY4wAVRBAEAAOCizNnya1c4Ly+mwsH9UQQBAADAbiVWm+ZtzZDEAqmoO3xq+wNFRUVauXKl9u/fr4KCAkVFRalz585KSkpyRnwAAABwYyv3ZivndIkig/3UvVmE2eEANVLjImjp0qV67bXXNGvWLJWUlCg8PFyBgYHKzs5WUVGRmjdvrt///vf6wx/+oNBQ2iICAAB4gjlb0iVJg9rHyJupcKgjajQdbuTIkRozZoyaNWumH3/8UXl5eTp+/LgOHTqkgoIC7dq1S08++aTmz5+v1q1ba968ec6OGwAAACaz2QzN3VI2FW5we6bCoe6o0UjQsGHD9PXXX8vX17fax5s3b67mzZtr/Pjx2rp1q9LT0x0aJAAAANzPugMndCyvSKEBPurTopHZ4QA1VqMi6O67767xAZOTk5WcnGx3QAAAAKgbZp9ZIPWqdjHy86HfFuqOWjdGOFtqaqoWLVokq9Wqvn37qmvXro6KCwAAAG7MMAzNOVME0RUOdY3dJfsbb7yhK6+8UosWLdLChQt1xRVX6LnnnnNkbAAAAHBTqYdzdfjkaQX6euuyVlFmhwPUSo1Hgg4ePKiEhISK719//XVt2bJFjRqVzf9cvny5Ro4cqb/+9a+OjxIAAABupbwr3OVtohTo521yNEDt1Hgk6KqrrtJrr70mwzAkSZGRkZozZ46KioqUl5enn376SVFRfAoAAADgCZgKh7qsxkXQ6tWrtWPHDvXs2VMbNmzQO++8o1deeUWBgYFq0KCBvvjiC3300UfOjBUAAABuYFdGnvYcOyU/by9d0Tba7HCAWqvxdLiwsDC9+eabWrZsmW677TZdccUV+uWXX2S1WmW1WtWgQQMnhgkAAAB3UT4K1K9VI4UGVL+ECuDOat0YoU+fPlqzZo0aNmyozp07a/HixRRAAAAAHqS8NfYQFkhFHVXjkaDS0lK988472rZtmzp16qQnnnhCY8aM0R/+8Ad9+OGHev311xUTE+PMWAEAAGCyA8cLtDU9V95eFl2VzHs/1E01HgmaMGGCXn/9dQUHB2vKlCl6+OGH1bp1ay1YsEBDhgxR79699dZbbzkzVgAAAJhs7payUaCeSRGKCPYzORrAPjUugr799lt9/fXXev755zVv3jx9//33FY9NmDBBK1as0C+//OKUIAEAAOAeZqeWtcamKxzqshoXQTExMfrxxx9VXFysBQsWKDIystLj0dHRmjZtmsMDBAAAgHvIyC3UugMnJUmDuR8IdViN7wl6/fXXdfPNN+uRRx5RXFycvvzyS2fGBQAAADdTPhWuS9MGigkLMDkawH41LoIGDhyojIwMZWVlsSgqAACABypvjT00Jc7kSICLU6sW2RaLhQIIAADAA2WfKtbKtGxJTIVD3VejImjIkCFasWLFBffLy8vTCy+8oDfeeOOiAwMAAID7+Glrhqw2Q8lxYWoaGWR2OMBFqdF0uBtvvFHXX3+9wsPDNWLECHXr1k2NGzdWQECATpw4oa1bt2rJkiX64YcfNGzYML344ovOjhsAAAAuNGdL+VQ4RoFQ99WoCJowYYLGjRun6dOn64svvtA777yjnJwcSWVT5JKTkzV48GCtXr1a7dq1c2rAAAAAcK28whIt2ZUlidbYqB9q3BjB399f48aN07hx4yRJOTk5On36tCIjI+Xr6+u0AAEAAGCuBdszVWy1qUVUsFrFhJodDnDRalwE/VZ4eLjCw8MdGQsAAADcUHlrbEaBUF/UqjscAAAAPMvpYqsWbj8mSRrSntbYqB8oggAAAHBOi3cd0+kSq5o0CFRKkzCzwwEcgiIIAAAA51S+QOqQlFhZLBaTowEcgyIIAAAA1SoutemnbRmSaI2N+sWuIujkyZN677339Pjjjys7u2zl4HXr1unw4cMODQ4AAADmWbYnS3mFpYoK9VeXpg3NDgdwmFp3h9u0aZOuuuoqhYeHa9++fbrrrrsUERGhGTNm6MCBA/r444+dEScAAABcrLwr3KDkGHl5MRUO9UetR4IeeeQR3Xbbbdq1a5cCAgIqtl999dVavHixQ4MDAACAOaw2Qz9uKZ8KR1c41C+1LoJWr16tu+++u8r2Jk2a6OjRow4JCgAAAOZavS9bx08VKzzQVz2bR5gdDuBQtS6C/P39lZubW2X7zp07FRUV5ZCgAAAAYK7yrnADk2Pk600vLdQvtX5Gjxw5Us8884xKSkokSRaLRQcOHNBf/vIXXX/99Q4PEAAAAK5lsxkV9wMNaU9XONQ/tS6CXnrpJeXn5ys6OlqnT59W//791bJlS4WGhuq5555zRowAAABwoU2Hc5SeU6hgP2/1a9XI7HAAh6t1d7jw8HDNmzdPS5cu1caNG5Wfn68uXbroqquuckZ8AAAAcLHyqXAD2kYrwNfb5GgAx6t1EfTxxx9rzJgx6tu3r/r27Vuxvbi4WJ9//rluvfVWhwYIAAAA1zEMQ3NS0yVJQ1ggFfVUrafD3X777crJyamyPS8vT7fffrtDggIAAIA5dmTkad/xAvn5eGlAm2izwwGcotZFkGEYsliqLpZ16NAhhYeHOyQoAAAAmGP25rKpcJe1ilKwf60nDQF1Qo2f2Z07d5bFYpHFYtGVV14pH59ff9RqtSotLU1DhgxxSpAAAABwjfKucEOZCod6rMZF0KhRoyRJGzZs0ODBgxUSElLxmJ+fn5o1a0aLbAAAgDosLeuUth/Nk4+XRVe2Yyoc6q8aF0ETJ06UJDVr1kxjxoxRQECA04ICAACA65V3hevdIlINgvxMjgZwnlpP9Bw/frwz4gAAAIDJ5pQvkMpUONRztS6CrFarXnnlFX355Zc6cOCAiouLKz2enZ3tsOAAAADgGkdOntbGgydlsUgDk2PMDgdwqlp3h5s8ebJefvlljRkzRjk5OXrkkUd03XXXycvLS5MmTXJCiAAAAHC28oYI3RMjFB3KbQ+o32pdBE2dOlXvvvuu/vSnP8nHx0djx47Ve++9p6efflorVqxwRowAAABwstln7gcazFQ4eIBaF0FHjx5Vhw4dJEkhISEVC6cOHz5c33//vWOjAwAAgNMdyyvS6n1ltzQMbs9UONR/tS6C4uPjlZ6eLklq0aKFfvzxR0nS6tWr5e/v79joAAAA4HQ/bcuQYUgd48MV3zDI7HAAp6t1EXTttddq/vz5kqT7779fTz31lFq1aqVbb71Vd9xxh92BPP/887JYLHrooYfsPgYAAABqr2IqXHumwsEz1Lo73PPPP1/x9zFjxigxMVHLli1Tq1atNGLECLuCWL16tf773/+qY8eOdv08AAAA7JNzukTLdmdJkoZyPxA8RK1Hgn6rV69eeuSRRzRixAitWbOm1j+fn5+vm2++We+++64aNmx4seEAAACgFuZvy1CpzVDrmBA1jwoxOxzAJWo9EpSfny9vb28FBgZWbNuwYYOeeuop/fDDD7JarbU63r333qthw4bpqquu0t/+9rfz7ltUVKSioqKK73NzcyVJJSUlKikpqdV5Ha38/GbHUReRO/uQN/uQN/uRO/uQN/uQN/vYk7fZm8vu9R7ULtqj881zzj7ulLfaxGAxDMOoyY4HDx7U6NGjtWrVKnl7e+u+++7T3/72N/3hD3/QF198oWuvvVYPP/ywevbsWeOTf/7553ruuee0evVqBQQE6PLLL9cll1yiV199tdr9J02apMmTJ1fZPm3aNAUFcRMfAABAbRRZpb+u9laJYdFjHUvVJNjsiAD7FRQU6KabblJOTo7CwsLOu2+NR4L+/Oc/q7CwUK+99ppmzJih1157Tb/88ot69uypPXv2KD4+vlZBHjx4UA8++KDmzZungICaLcj1+OOP65FHHqn4Pjc3VwkJCRo0aNAFL9TZSkpKNG/ePA0cOFC+vr6mxlLXkDv7kDf7kDf7kTv7kDf7kDf71DZvs1OPqmTVJiU0DNSdN/STxWJxQZTuieecfdwpb+WzxGqixkXQ4sWLNWPGDPXq1UujR49WbGysbr75Zru7ua1du1aZmZnq0qVLxTar1arFixfr9ddfV1FRkby9vSv9jL+/f7VtuH19fU1Pejl3iqWuIXf2IW/2IW/2I3f2IW/2IW/2qWneftpe1hDh6g5x8vPzc3ZYdQLPOfu4Q95qc/4aF0EZGRlKSkqSJEVHRysoKEhDhw6tfXRnXHnlldq8eXOlbbfffrvatm2rv/zlL1UKIAAAADhOUalVC7ZnSpIG0xUOHqZWjRG8vLwq/f1iPjEIDQ1VSkpKpW3BwcGKjIyssh0AAACOtXR3lvKLShUbFqBL4huYHQ7gUjUuggzDUOvWrSvmiubn56tz586VCiNJys7OdmyEAAAAcLg5FQukxsjLy3PvBYJnqnERNGXKFGfGIUn6+eefnX4OAAAAT1dqtWne1gxJTIWDZ6pxETR+/HhnxgEAAAAXWZWWrRMFJYoI9lOPZhFmhwO4nNeFdwEAAEB9MvvMVLiB7WLk483bQXgenvUAAAAexGYzNHdLWRE0pANT4eCZKIIAAAA8yPqDJ5WZV6RQfx/1aRFpdjiAKSiCAAAAPMic1HRJ0hXtouXvw7qM8EwUQQAAAB7CMAzNOTMVbihd4eDBarVYqiRZrVZ9+OGHmj9/vjIzM2Wz2So9vmDBAocFBwAAAMfZciRXB7NPK8DXS5e1jjI7HMA0tS6CHnzwQX344YcaNmyYUlJSKhZPBQAAgHsrb4hweetoBfnV+m0gUG/U+tn/+eef68svv9TVV1/tjHgAAADgJOWtsYcwFQ4ertb3BPn5+ally5bOiAUAAABOsjszT7sz8+XrbdEV7aLNDgcwVa2LoD/96U967bXXZBiGM+IBAACAE8zdkiFJ6tuykcICfE2OBjBXrafDLVmyRAsXLtTs2bPVvn17+fpWfhHNmDHDYcEBAADAMWafaY09pD1T4YBaF0ENGjTQtdde64xYAAAA4AQHswuUejhXXhZpYHKM2eEApqt1ETRlyhRnxAEAAAAnKe8K1yMpQpEh/iZHA5jP7t6Ix44d044dOyRJbdq0UVQUveYBAADc0ZzU8gVS40yOBHAPtW6McOrUKd1xxx2Ki4vTZZddpssuu0yNGzfWhAkTVFBQ4IwYAQAAYKfM3EKtPXBCkjSoPVPhAMmOIuiRRx7RokWLNGvWLJ08eVInT57Ut99+q0WLFulPf/qTM2IEAACAneZuzZBhSJckNFBceKDZ4QBuodbT4b7++mt99dVXuvzyyyu2XX311QoMDNTo0aP11ltvOTI+AAAAXIS5FVPh6AoHlKv1SFBBQYFiYqoOpUZHRzMdDgAAwI2cOFWs5XuPS5KGUAQBFWpdBPXu3VsTJ05UYWFhxbbTp09r8uTJ6t27t0ODAwAAgP1+2pYhq81Qu7gwJUYGmx0O4DZqPR3utdde0+DBgxUfH69OnTpJkjZu3KiAgADNnTvX4QECAADAPuWtsVkgFais1kVQSkqKdu3apalTp2r79u2SpLFjx+rmm29WYCA32wEAALiD/KJSLd6VJYmpcMBv2bVOUFBQkO666y5HxwIAAAAHWbg9U8WlNjVvFKzWMSFmhwO4lRoVQTNnztTQoUPl6+urmTNnnnffkSNHOiQwAAAA2K98gdTBKbGyWCwmRwO4lxoVQaNGjdLRo0cVHR2tUaNGnXM/i8Uiq9XqqNgAAABgh8ISqxbuyJREa2ygOjUqgmw2W7V/BwAAgPtZuvu4CoqtatIgUB2ahJsdDuB2at0i++OPP1ZRUVGV7cXFxfr4448dEhQAAADsN3drhiRpcHumwgHVqXURdPvttysnJ6fK9ry8PN1+++0OCQoAAAD2sdqk+duPSaIrHHAutS6CDMOo9hOFQ4cOKTyc4VYAAAAzWG2GVqZl64eDXsotLFVksK+6JjY0OyzALdW4RXbnzp1lsVhksVh05ZVXysfn1x+1Wq1KS0vTkCFDnBIkAAAAzm1Oaromz9qq9JxClX/GfbrEpnlbj2pISpy5wQFuqMZFUHlXuA0bNmjw4MEKCfm137yfn5+aNWum66+/3uEBAgAA4NzmpKbrnk/XyfjN9oJiq+75dJ3eGteFQgj4jRoXQRMnTpQkNWvWTGPGjFFAQIDTggIAAMCFWW2GJs/aWqUAOtvkWVs1MDlW3l40SADK1fqeoPHjx1MAAQAAuIFVadlnpsBVz5CUnlOoVWnZrgsKqANqPBJUzmq16pVXXtGXX36pAwcOqLi4uNLj2dm8yAAAAFwhM+/cBZA9+wGeotYjQZMnT9bLL7+sMWPGKCcnR4888oiuu+46eXl5adKkSU4IEQAAANWJDq3Z7Jya7gd4iloXQVOnTtW7776rP/3pT/Lx8dHYsWP13nvv6emnn9aKFSucESMAAACq0SMpQnHhATrX3T4WSXHhAeqRFOHKsAC3V+si6OjRo+rQoYMkKSQkpGLh1OHDh+v77793bHQAAAA4J28viyaOSK62MUJ5YTRxRDJNEYDfqHURFB8fr/T0dElSixYt9OOPP0qSVq9eLX9/f8dGBwAAgPMa3D5WiZFBVbbHhgfQHhs4h1o3Rrj22ms1f/589ezZU/fff7/GjRun999/XwcOHNDDDz/sjBgBAABwDmv2n9D+4wXy9bbo1dEdtXLNOg26tKd6t4xmBAg4h1oXQc8//3zF38eMGaOmTZtq+fLlatWqlUaMGOHQ4AAAAHB+7/+SJkm6oWu8BiXHqHSfoZ5JERRAwHnUugj6rd69e6t3796OiAUAAAC1cOB4geZuPSpJuqNvksnRAHVHjYqgmTNn1viAI0eOtDsYAAAA1NyUZWkyDOmy1lFqFROqkpISs0MC6oQaFUGjRo2q0cEsFousVuvFxAMAAIAayC0s0ZerD0qS7uzHKBBQGzUqgmw2m7PjAAAAQC18seqgThVb1TomRJe2amR2OECdUqMW2RERETp+/Lgk6Y477lBeXp5TgwIAAMC5lVpt+nDZPknShH5JslhoggDURo2KoOLi4opFUT/66CMVFhY6NSgAAACc25wtR3X45GlFBvvpmkuamB0OUOfUaDpc7969NWrUKHXt2lWGYeiBBx5QYGBgtft+8MEHDg0QAAAAlb13pi32uF6JCvD1NjkaoO6pURH06aef6pVXXtGePXtksViUk5PDaBAAAIAJ1u4/oQ0HT8rP20vjeiWaHQ5QJ9WoCIqJialYJDUpKUmffPKJIiMjnRoYAAAAqnp/yV5J0qjOjRUV6m9yNEDdVOvFUtPS0pwRBwAAAC7gYHaB5qSeWRyVttiA3WpdBEnS/PnzNX/+fGVmZlZpn809QQAAAM7x4bJ9shnSpa0aqW1smNnhAHVWrYugyZMn65lnnlG3bt0UFxdHS0YAAAAXyCss0RdnFkdlFAi4OLUugt5++219+OGHuuWWW5wRDwAAAKrxxeqDyi8qVcvoEPVvFWV2OECdVqN1gs5WXFysPn36OCMWAAAAVOPsxVHv6JskLy9m4gAXo9ZF0J133qlp06Y5IxYAAABU48etGTp04rQaBvnqui4sjgpcrFpPhyssLNQ777yjn376SR07dpSvr2+lx19++WWHBQcAAADp/SUsjgo4Uq2LoE2bNumSSy6RJKWmplZ6jCYJAAAAjrX+wAmt3X9Cft5euqU3i6MCjlDrImjhwoXOiAMAAADVKB8FGtGpsaJDA0yOBqgfan1PEAAAAFzj8MnTmn1mcdQJtMUGHKbGI0HXXXddjfabMWOG3cEAAADgVx8t2yerzVCfFpFKbsziqICj1LgICg8Pd2YcAAAAOEt+Uak+W3lAknTnpYwCAY5U4yJoypQpzowDAAAAZ5m+5qDyikrVPCpYl7eONjscoF7hniAAAAA3Y7UZ+mBpWUMEFkcFHI8iCAAAwM3M25qhg9mn1SDIV9d3iTc7HKDeoQgCAABwM+8v2StJurlnUwX6sTgq4GgUQQAAAG5k48GTWr3vhHy9Lbq1dzOzwwHqJYogAAAAN1KxOGrHxooJY3FUwBkoggAAANzEkZOn9cPmdEnSHSyOCjgNRRAAAICb+Gj5PpXaDPVqHqGUJqzRCDgLRRAAAIAbOHXW4qgT+jU3ORqgfqMIAgAAcANfrT2k3MJSNYsM0pVtWRwVcCaKIAAAAJNZbYamlC+O2o/FUQFnowgCAAAw2fxtGdp3vEDhgb66oSuLowLORhEEAABgsvK22GN7NFWQn4/J0QD1H0UQAACAiVIP52hlWrZ8vCwa3yfR7HAAj0ARBAAAYKLyUaBhHeMUFx5ocjSAZ6AIAgAAMMnRnELN2nhEkjSBxVEBl6EIAgAAMMnHZxZH7dEsQh3jG5gdDuAxKIIAAABMUFBcqmmryhZHvYNRIMClKIIAAABM8PW6wzpZUKKmEUEamBxjdjiAR6EIAgAAcDGbzdCUMw0Rbu/bTN4sjgq4FEUQAACAiy3ckam9WacUGuCjG7slmB0O4HEoggAAAFzs7MVRQ/xZHBVwNVOLoLfeeksdO3ZUWFiYwsLC1Lt3b82ePdvMkAAAAJxqy5EcLdtzXN5eFo3v08zscACPZGoRFB8fr+eff15r167VmjVrdMUVV+iaa67Rli1bzAwLAADAaT5Ysk+SNDQlVk0asDgqYAZTx19HjBhR6fvnnntOb731llasWKH27dubFBUAAIBzZOYWaubGw5KkOy9tbnI0gOdym0moVqtV06dP16lTp9S7d+9q9ykqKlJRUVHF97m5uZKkkpISlZSUuCTOcyk/v9lx1EXkzj7kzT7kzX7kzj7kzT71NW8fLk1TidVQl6YN1D422OHXV1/z5grkzj7ulLfaxGAxDMNwYiwXtHnzZvXu3VuFhYUKCQnRtGnTdPXVV1e776RJkzR58uQq26dNm6agoCBnhwoAAGC3Yqs0aZ23TpVadHtrqy6JNPUtGFDvFBQU6KabblJOTo7CwsLOu6/pRVBxcbEOHDignJwcffXVV3rvvfe0aNEiJScnV9m3upGghIQEZWVlXfBCna2kpETz5s3TwIED5evra2osdQ25sw95sw95sx+5sw95s099zNvnqw/pqZlbFd8gQD89fKlT1gaqj3lzFXJnH3fKW25urho1alSjIsj06XB+fn5q2bKlJKlr165avXq1XnvtNf33v/+tsq+/v7/8/f2rbPf19TU96eXcKZa6htzZh7zZh7zZj9zZh7zZp77kzWYz9OHy/ZKk2/s1V4C/n1PPV1/yZgZyZx93yFttzu926wTZbLZKoz0AAAB13aJdx7Tn2CmF+PtodLd4s8MBPJ6pI0GPP/64hg4dqqZNmyovL0/Tpk3Tzz//rLlz55oZFgAAgEO9/0vZ4qi/656g0ABGGQCzmVoEZWZm6tZbb1V6errCw8PVsWNHzZ07VwMHDjQzLAAAAIfZfjRXS3ZnycsiFkcF3ISpRdD7779v5ukBAACcrnwUaGhKnBIi6GYLuAO3uycIAACgvjiWV6RvNxyRJN3RL8nkaACUowgCAABwkk9W7Fex1abOTRuoa2JDs8MBcAZFEAAAgBMUllg1dUVZW+wJjAIBboUiCAAAwAn+t/6wjp8qVpMGgRrSPtbscACchSIIAADAwQzD0PtLyhoi3NanmXy8ecsFuBNekQAAAA62eFeWdmXmK9jPW2N6JJgdDoDfoAgCAABwsPJRoNHdExTG4qiA26EIAgAAcKCdGXlavPOYvCzS7X1oiAC4I4ogAAAAB/rgzCjQoORYNY1kcVTAHVEEAQAAOEhWfpFmrD8sSbrzUkaBAHdFEQQAAOAgU1ccUHGpTZ3iw1kcFXBjFEEAAAAOUFhi1Scr9kmSJlzaXBaLxdyAAJwTRRAAAIADzNx4RFn5xYoLD9DQFBZHBdwZRRAAAMBFMgyjoiHCbX2ayZfFUQG3xisUAADgIi3dfVzbj+YpyM9bv+vR1OxwAFwARRAAAMBFem/JXknS6G4JCg9kcVTA3VEEAQAAXITdmXn6eccxWSzS7X2bmR0OgBqgCAIAALgIHyzdJ0m6ql2MEiODzQ0GQI1QBAEAANgp+1Sxvl57SJJ0Zz8WRwXqCoogAAAAO01buV9FpTalNAlTj6QIs8MBUEMUQQAAAHYoKrXqo+X7JUl39mNxVKAuoQgCAACww3cb03Usr0gxYf66ukOc2eEAqAWKIAAAgFoyDEPvnVkcdXyfZvLz4S0VUJfwigUAAKil5XuPa1t6rgJ9vXUTi6MCdQ5FEAAAQC29/0vZKNANXePVIMjP5GgA1BZFEAAAQC3sPZav+dszJbE4KlBXUQQBAADUwgdLy0aBrmoXreZRISZHA8AeFEEAAAA1dLKgWF+dWRz1DhZHBeosiiAAAIAamrrygApLbEqOC1Pv5pFmhwPAThRBAAAANVBcatPHy/dJkib0S2JxVKAOowgCAACoge83H1FGbpGiQ/01olNjs8MBcBEoggAAAC7AMAy9f2Zx1Ft7J7I4KlDH8QoGAAC4gJVp2Uo9nKsAXy/d1DPR7HAAXCSKIAAAgAsoHwW6rku8IoJZHBWo6yiCAAAAzmNf1in9tC1DknRHX9piA/UBRRAAAMB5TFmaJsOQBrSJUstoFkcF6gOKIAAAgHPIKSjRl2vKFke989LmJkcDwFEoggAAAM7hs9UHdLrEqraxoerTgsVRgfqCIggAAKAaJVabPly6TxKLowL1DUUQAABANX7YnK6juYVqFOKvkZewOCpQn1AEAQAA/MZvF0f19/E2OSIAjkQRBAAA8Btr9p/QpkM58vPx0s09m5odDgAHowgCAAD4jfd+2StJur5LE0WG+JscDQBHowgCAAA4y/7jp/TjVhZHBeoziiAAAICzTFm6T4Yh9W8dpVYxoWaHA8AJKIIAAADOyDldoulrDkoqa4sNoH6iCAIAADjji9UHdKrYqtYxIbq0VSOzwwHgJBRBAAAAkkpZHBXwGBRBAAAAkmanHtWRnEJFBvvpmkuamB0OACeiCAIAAB7PMAy9d2Zx1HG9EhXgy+KoQH1GEQQAADzeugMntPHgSfn5eGlcr0SzwwHgZBRBAADA471/ZhRo1CWNFRXK4qhAfUcRBAAAPNrB7ALNST0qSbqDttiAR6AIAgAAHu3DZftkM6RLWzVS29gws8MB4AIUQQAAwGPlFZboi9Vli6MyCgR4DoogAADgsb5YfVD5RaVqGR2i/q2izA4HgItQBAEAAI9UarXpw2X7JEl39E2SlxeLowKegiIIAAB4pB+3ZujQidNqGOSr67qwOCrgSSiCAACAR3qfxVEBj0URBAAAPM76Aye0dv8J+Xl76ZbeLI4KeBqKIAAA4HHKR4FGdGqs6NAAk6MB4GoUQQAAwKMcPnlas88sjjqBttiAR6IIAgAAHuWjZftktRnq0yJSyY1ZHBXwRBRBAADAY+QXleqzlQckSXdeyigQ4KkogoA6yGoztDItW2uzLFqZli2rzTA7JADV4LXqfqavOai8olI1jwrW5a2jzQ4HgEl8zA4AQO3MSU3X5FlblZ5TKMlbH+9ao7jwAE0ckawhKXFmhwfgDF6r7sdqM/TB0rKGCCyOCng2RoKAOmROarru+XTdmTdVvzqaU6h7Pl2nOanpJkUG4Gy8Vt3TvK0ZOph9Wg2CfHV9l3izwwFgIoogoI6w2gxNnrVV1U2mKd82edZWptsAJimx2pSZV6gtR3L0xDepvFbd0PtL9kqSbu7ZVIF+LI4KeDKmwwF1xKq07CqfKp/NkJSeU6hVadnq3SLSdYEB9ZBhGCootir7VHHF1/FTxTrxmz+zTxXpREGJjucXKbewtGbHFq9VM2w8eFKr952Qr7dFt/ZuZnY4AExGEQTUEek5p2u0X2beuQslwFNZbYZOFhSfu6g589jx/LK/Hz9VrOJSW63PY7FIQb7eOlVsveC+vFZdq2Jx1I6NFRPG4qiAp6MIAtxcTkGJPlt9QO8s3lOj/ZfsylLv5pGK5j95ONDZXc4i07LVu2W0vE28qbywxFo2EpNfrOyCshGZ7FMlZ/4srvJ18nSJDDtmn/n5eCky2E8Rv/0K8lNEiJ8ig/3UMMhPkSFlfzYI8tOqtGyNfXfFBY994lSxHVcOexw5eVo/bC67D+sOFkcFIIogwG3tyzqlKUvTNH3tIRWc+VTZyyJd6DaC6WsP6Zv1hzW4faxu7tVUvZtHymKhAxLs5+wuZzabodzCkjPTy2r2dbrkwiMt1QkP9K22mIkIOvP92X8P9lOQn3etXz89kiIUFx6gozmF1d4XVG7SrK3afDhX/ze0raJC/e26HtTMR8v3qdRmqFfzCKU0CTc7HABugCIIcCOGYWjF3my9vyRN87dnVHxy3TY2VHf0S1KAj5ce/HxD2b5n/Vz5W7Tb+jbT5kM5WrP/hL7fnK7vN6erRVSwbu6ZqOu7xis80NeVl4N6oLzL2W/fzJd3OXtrXJcqhVBRqVUnTpXo+G9GZc6eenY8/8y2gmKdKCixq0mAr7dFEWeNxEQE+ysiyLfsz+CyPxsG+yoy2F8RwX5qEOQrX2/n9wPy9rJo4ohk3fPpOllU/Wu1b8tILd1zXF+vO6Qftx7Vo4PaaFyvRFNH1+qrU2ctjjqhX3OTowHgLiiCADdQXGrTd5uO6P0ladpyJLdi+4A2UZrQr7n6tvx1NMfPx+usT+XLxP7mU/lt6bmaunK/vll3WHuOndIz323VP+du1zWdmmhcr0R1iOeTUFxYTToSPvzFRn2x+qCyC8qmop04VaL8opo1CPitUH8fRZyZVvbbKWgNg89MPQv+9bEQfx+3HeUckhKnt8Z1Oe9rdf2BE3rq21SlHs7VxJlb9OWag3rmmhR1TWxoYuT1z1drDym3sFTNIoN0ZVsWRwVQhiIIMNGJU8WaunK/Pl6+X5l5RZKkAF8vXdclXnf0TVLL6JAqPzMkJU4Dk2O1fHemfvxlpQZd2rPK/Rnt4sL0t1Ed9H9D2+mb9Yc1dcV+bT+apy/WHNQXaw6qU3y4bu6VqBEdG9MmFue0cu/x83YklKTTJVYt3HGsynZvL0tFMVM+GtPwzOjM2cXM2ffT+PnUr1UbLvRa7dy0ob69t5+mrTqgF+ds15Yjubr+rWUa3S1efxnSVpEhTJG7WFaboSnli6P2Y3FUAL+iCAJMsDszXx8sTdOMdYdUWFLWgSo61F/j+zTTTT2aqmGw33l/3tvLop5JETq+zVDPpIhzTqEJ8ffRLb0SNa5nU63df0KfrtivHzYf1cZDOdr41SY99/023dA1Xjf3bKrmUVULLngewzC0/uBJzdp4RF+vO1SjnxnbI0ED2kRXFDORwf4KDfDhDacu/Fr19rLoll6JGpoSqxdmb9f0tYf05ZpDmrslQ48NaaPfdW/KFLmLMH9bhvYdL1B4oK9u6MriqAB+RREEuIhhGFqyO0vvL0nTz2d9ct6+cZjuvDRJwzo0dton4RaLRd2aRahbswg9NbxIX645pGmr9utg9mm9vyRN7y9JU7+WjTSuV1Nd1S5GPi64bwLuwzAMbTmSq1mbjui7jek6fLJm7djLjezUhPVuLlKjEH+9eGMnjemeoKe+3aJt6bn66zep+mL1QT17TYo6JTQwO8Q6qbwt9tgeTRXkx1seAL/iXwTAyQpLrJq54Yg+WJqm7UfzJJWtJXJVuxhN6JeknkkRLr2vITLEX/dc3kJ3X9Zci3Yd06fL92vBjkwt2Z2lJbuzFBPmr991b6qxPZoqNpw22/XZzow8zdp4RN9tSlda1qmK7UF+3hqYHKNhKXF6emaqMnKLqr0vyKKye1x6JEW4LOb6rluzCM26r68+XbFfL/24U5sO5WjUm0s1tkdT/XlQmwuOEuNXqYdztDItWz5eFo3vk2h2OADcDEUQ4CRZ+UX6dMV+fbpiv7Lyy9YDCfLz1o1d43V73yQ1axRsanxeXhYNaBOtAW2idehEgT5bdUBfrD6ojNwivTZ/l15fuFsD28VoXK9E9WkRydSmeiIt65S+23hEszYd0c6M/Irt/j5eurJdtIZ3bKwBbaIr7hWzyThvl7OJI5KZruVgPt5euq1vkq7uGKfnf9iuGesPa9rKA5q9OV3/N7StbuyawOuxBspHgYZ1jFNceKDJ0QBwNxRBgIPtOJqn95fs1f82HKlYcb5xeIDG92mm33VvqvAg92tTHd8wSH8e3FYPXtlac7cc1Scr9mtVWrbmbDmqOVuOKqlRsG7u2VQ3dI1XgyA+ia5rDp0o0Heb0vXdpiNKPfxr90Ffb4v6t47SiE6NdWW7GIX4V/0voSZdzuAc0aEBennMJRrTPUFPf7tFOzLy9JevN+uzVQf1t1EprHdzHkdzCjVr4xFJ0gQWRwVQDYogwAFsNkOLdh3TB0vS9MuurIrtnRIaaEK/JA1NiXXJ+iQXy8/HSyM6NdaITo21MyNPU1fs19frDist65T+9v02vTh3h0Z0aqxxvRLVKT7cbdsTQ8rILdT3m9I1a9MRrT9wsmK7t5dFfVs20vCOcRqcHFujorwmHQnhPD2bR+q7B/rpo2X79Mq8ndpw8KRGvr5E43ol6k8D27jlBytm+/jM4qg9mkWoY3wDs8MB4IYogoCLcLrYqhnrD+mDJWnac6zsngovizQkJVYT+iWpS9OGdbZQaB0TqsnXpOixIW317YYj+nTFfm1Nz9VXaw/pq7WHlNIkTON6JmrkJY254dhNHM8v0g+pR/XdxiNatS+7YrFdi0XqmRShEZ0aa0j7WLtaL9e0IyGcw9fbS3de2lwjOjXWc99v08yNR/Tx8v36flPZFLnru8QzRe6MguJSTS1fHPVSRoEAVI93LoAdMnML9fHy/Zq6cr9OFJRIKmtHPaZ7gm7r00wJEUEmR+g4wf4+uqlnU43tkaD1B0/q0xX79d2mdKUeztX/zdis537Ypuu7xGtcr6ZqGR1qdrgeJ6egRHO3HNWsTUe0bM9xWW2/3rnTpWkDjejUWFd3iFNMGE0u6oOYsAD9e2xn/a57gp6euUW7M/P15682lXWRG5WidnFhZodouq/XHVbO6RI1jQjSVe1izA4HgJuiCAJqIfVwjj5YkqZZm46oxFr2ZjO+YaBu75uk0d3iFRpQf6elWCwWdWnaUF2aNtRTw5I1fe1BTV15QPuPF+jDZfv04bJ96tU8QuN6JWpQcmy9W/jSneQXlWre1qP6bmO6Fu86VvFclKQOTcI1vGOchnWMU3zD+lOMo7I+LRvphwcu1QdL0/Tv+bu0Zv8JDf/PEt3aO1EPD2ytsHr8b9H52GyGPjjTEOGOvs0YsQRwTqYWQf/4xz80Y8YMbd++XYGBgerTp49eeOEFtWnTxsywgEpsNkPzt2fq/SV7tWJvdsX2bokNNaFfkga1j/W4/2gbBvvp95e10J39muuX3Vn6dMV+zd+WoRV7s7Vib7aiQv31u+4JGtujqRo3oCuTI5wutmrB9kx9t+mIFmzPVNGZphuS1CYmVCM6xWl4x8amdx2E6/j5eOkP/Vto5Jkpct9vTteUpfv03aZ0/fXqdrrmksZ1djquvRbuyFRa1imFBvjoxm4JZocDwI2ZWgQtWrRI9957r7p3767S0lI98cQTGjRokLZu3argYP4jh7lOFZXq63Vl9/vsO14gqey+iKs7xGlCvyRdwuKF8vIq6y7Wv3WUjpw8rc9XHdBnqw/qWF6R/rNgt95YuFtXnmmzfWnLRtyzUEtFpVYt3pmlWRuP6KdtGSootlY81rxRsIZ3jNPwTo3VOoZpiJ6scYNAvXFzF43ZeUyTZm7R3qxTeuiLDZq26oCevSZFbWI95/nx3i9lo0A39Wiq4Gq6HQJAOVP/hZgzZ06l7z/88ENFR0dr7dq1uuyyy6rsX1RUpKKioorvc3PLWr2WlJSopKTEucFeQPn5zY6jLnK33KXnFOqTFQf0xZpDyi0slSSFBfhoTLd43dKrqeLOLCBqdrzulreoYB/dP6C5/nBZM/20LVPTVh3UirQTmrc1Q/O2ZqhpRKB+1z1e13duoggTF3x0t7z9VonVpuV7s/X95qOaty1TeWeeg5LUpEGAhnWI1dUpsUqOC634lN9V1+LuuXNXrspb76QGmnlvb01Zuk9vLNqrVWnZuvrfv+i23k1134AW1bZAd2e1zdvW9Fwt33tc3l4W3dwj3mOfp7xO7Ufu7ONOeatNDBbDMKpbCNwUu3fvVqtWrbR582alpKRUeXzSpEmaPHlyle3Tpk1TUBBz33Fx9udLPx/x0objFtnOLAXZKMDQ5XE29Ygy5O9tcoB1UMZpaelRL606ZtFpa1lOfSyGOkca6htrU7OQss5lns5mSHtyLVqXZdHGbItOlf6alHBfQ5c0MtQl0qZE8oUayi6SvtnnpU3ZZffmhfsaGtXMps6RRr19Dn2620urj3mpc6RNt7W2XfgHANQ7BQUFuummm5STk6OwsPM3inGbIshms2nkyJE6efKklixZUu0+1Y0EJSQkKCsr64IX6mwlJSWaN2+eBg4cKF9fz7wh1V5m5s5qMzRvW6Y+XLZfa89aS6VnUkPd3jtRl7eJctv7ferSc66guFTfbz6qaasOKfXIr4t1to0N1U094jWyY5zLpq64S95sNkPrD57U96kZmpN6VMfyiyseiwj21dD2sbq6Q4y6NW3oNtMI3SV3dY2ZeVu085ie+X67DmSfliT1bh6hp4e1VcvoEJfGYY/a5C0zr0iXv7RYJVZDX93dU53iPXchWV6n9iN39nGnvOXm5qpRo0Y1KoLcZmz83nvvVWpq6jkLIEny9/eXv3/V9S18fX1NT3o5d4qlrnFl7vIKS/TlmkOasjRNh06UvTnw9bZoRMfGuqNfUp1aib0uPOfCfX11U68k3dQrSRsPntQnK/Zr1sYj2n40T0/P3KZ/zt2l67o00bheiS67v8WMvBmGoc2HczRr4xF9vyldR3IKKx4LD/TVkPaxGtGpsXo1j5CPGy+uWxeec+7IjLxd1b6x+rWO0TuL9+qNhbu1fG+2Rr65XBP6NdcDV7asE2t81SRvn63eqxKroa6JDdUtqZGLInNvvE7tR+7s4w55q8353eJfv/vuu0/fffedFi9erPj4eLPDQT12MLusnfMXqw8qv6jsXouGQb66uWeibumdyFoqLtApoYE6JTTQk8Pa6au1hzRt5QHtzTqlj5fv18fL96tHswiN652oIe3rR5ttwzC0/Wievtt0RLM2putAdkHFYyH+PhqUHKPhneLUr2VUvbheuJ8AX289cGUrjbqkiSbP2qL52zP19qI9mrnhsJ4anqwhKbF1uotcYYlVU1fulyTd2Y/FUQHUjKlFkGEYuv/++/XNN9/o559/VlIS/3jB8QzD0LoDJ/T+kjTNST2q8rUkW0QF645+Sbquc7wC/bjhx9UaBPnpzkuba0K/JC3bc1yfLN+vedsytGpftlbty1ajED+N7lbWZrsuLj67OzNf3206ou82pWt3Zn7F9gBfL13ZLkYjOjbW5W2iFODLcw+u0TQySO/f1l0/bc3QpFlbdOjEad0zdZ0ubdVIz1yToqQ62l59xrrDOlFQoviGgRrUPtbscADUEaYWQffee6+mTZumb7/9VqGhoTp69KgkKTw8XIGBrC2Ci1NitWl26lG9vyRNGw+erNh+aatGuqNfkvq3inKbey08mcViUd+WjdS3ZSMdzSnU56sP6LNVB5SRW6Q3f96jtxbt0YA20bqlV6Iua+2+92hJZSONs86M+GxL//XeJz9vL13eJkrDOzXWlW2jad0LU12VHKN+rRrpzYW79faivfplV5YGv7JYv7+sue4d0LJOfShksxl6f8leSdLtfZPc+t8HAO7F1P+J33rrLUnS5ZdfXmn7lClTdNttt7k+INQLOadL9PmqA/po2b6Key78fLw06pKy+33axprbRAPnFhseoIeuaq17B7TU/G0Z+nTFAS3ZnaUF2zO1YHum4hsG6qaeTTW6W4IahVS9P9AM6Tmn9f2mdM3alF6p2Pbxsqhfq0Ya0bGxBraPUVgA88vhPgJ8vfXIoDa6rku8Js7cokU7j+n1hbv1zfrDmjgiWQOTY+rEFLlFu45pz7FTCvH30ehuTKcHUHOmT4cDHGVf1ilNWZqm6WsPVSwqGRnsp1t6J2pcr0S3edOMC/P19tKQlDgNSYnT3mP5mrbygKavPaRDJ07rn3N26JV5O3V1hziN65WobokNXf5m7VhekWanpmvWxiNave9ExXYvi9S7RaSGd2ysIe1j1dDE9ZCAmmjWKFgf3t5dc7dk6NnvturwydP6/SdrNaBNlCaNbK/ESPeeIvf+mcVRf9c9QaF80ACgFpiTgTrNMAytTMvW+0vS9NO2DJXX1W1iQjWhX5JGXtKYey7quOZRIXpyeLIeHdxGszYe0acrD2jjwZP6dsMRfbvhiNrEhGpcr6Ya1bmJU98EnThVrDlbjuq7TUe0fM/xinvLJKl7s4Ya0amxhqbEKSqUYht1i8Vi0ZCUWF3WupHeWLhb7yzeq4U7jmnpK4t1T/8WuufyFm757+j2o7lasjtLXhZpfJ9mZocDoI6hCEKdVFxq0/ebj+i9X9K05ax1Zy5vE6U7+zVX35aRdWIqB2ouwNdbN3ZL0I3dEpR6OEefrtiv/204rB0ZeXrq2y16fvZ2jepc1ma7XZxjpjzmFpZo3pYMzdp0REt2Zan0rMqnU0IDjegYp6s7xKlxA+5hRN0X5OejPw9uWzZF7tstWrI7S6/N36Vv1h/WpJHJuqJtjNkhVlI+CjQ0Ja5ONk8BYC6KINQpJ04Va9qZ+30y88oWzg3w9dJ1XeJ1R99mahntmjVmYK6UJuF6/vqOevzqdpqx7pA+XbFfe46d0tSVBzR15QF1TWyocb2aamhKXKVPsK22spHDtVkWRaZlq3fL6Co3UhcUl+qnbZn6buMR/bzzmIpLf115vl1cmEZ0itPwDo3VNJI3XaifWkSF6JMJPfTD5qN69rutOpBdoDs+XKOByTF6eniyWxQcx/KK9O2GI5KkO2iLDcAOFEGoE/Ycy9cHS9L09bpDKiwpe1MaHeqv8X2a6aYeTbn3wkOFB/rq9r5Juq1PM63Ym61PV+zX3C1HtXb/Ca3df0LPfrdNN3aL1809ErU1PUeTZ21Vek6hJG99vGuN4sIDNHFEsi5vE62fdxzTrE1HtGBbpk6XWCvO0SIqWCM6Ndbwjo3VMjrEvIsFXMhisWhYxzhd3iZK/56/S+8vSdO8rRlavPOY7hvQUr/v31z+PuZNkftkxX4VW23q3LSBuiY2NC0OAHUXRRBMdb5P5g3D0NLdx/X+krL56eXaNw7ThH5JGt6xMYtLQlLZG7beLSLVu0WkMnML9cXqg/ps1QEdySnUfxft1X8X7a3259JzCvWHT9cpwMdLhWeN+DSNCCob8enYWG1jQ5laCY8V7O+jx69upxu6xuupb1O1Ym+2Xpq3U1+vO6TJ16Sof+sol8dUWGLV1BVli6NOYBQIgJ0ogmCaOanp1X4y//jQtiostemDJWnafjRPkmSxSFe2jdGEfknq1TyCN6U4p+iwAN1/ZSvdc3kLLdxxTB8v36dfdmWd92cKS22KC/PX8E6NNaJTY3VoEs5zDDhLq5hQfXZXL83ceETPfb9N+44XaPwHqzSkfayeGpGsJi68L+5/6w/r+KliNWkQqCEsjgrAThRBMMWc1HTd8+k6/bZJenpOoR74fEPF94G+3hrdLV639U2qs6uZwxw+3l4amByjEH+fCxZBkvTS6EvUp2UjF0QG1E0Wi0XXXNJEV7SN1qs/7dKHy/ZpzpajWrTzmO6/sqXu7Nfc6aPzhmHo/SVlDRFu69NMPt7MBgBgH4oguJzVZmjyrK1VCqCzeVmkRwe30c09EhUexNoPsF9mXmGN9juWX+TkSID6ITTAV08NT9aN3eL19P+2aNW+bP1zzg59tfaQnr0mRX2d+GHC4l1Z2pWZr2A/b43pkeC08wCo/yiC4HCFJVZl5RcpK79YWXlFZ/5e9v2xvCLtOZZ/ZgrcudkMqXNCQwogXLTo0ACH7gegTNvYMH1xdy99s/6w/v7DNu09dko3v7dSwzrG6alhyYoNd/xrqnwUaHT3BIWxOCqAi0ARhBopLLHqWN6vxUxWftFZ3xcpK+/Mtvwi5RWWOuScNf0EHzifHkkRigsP0NGcwmpHHy2SYsMD1CMpwtWhAXWexWLRdV3idWW7GL0yb6c+Xr5P329K18/bM/XgVa10e98k+TpoytrOjDwt3nlMXhbp9j40RABwcSiCPNjpYmtF4ZKVV/5n8VkjN7+O3uQX1a6w8fP2UqMQPzUK9VejEP+yv4eU/f1kQbH+vWD3BY/BJ/NwBG8viyaOSNY9n66TRapUCJW3Ppg4IrnKekEAai480FeTRrbXjd3i9dT/UrXuwEn9/Yftmr7mkJ65JkW9W0Re9Dk+ODMKNCg5lnW6AFw0iiAHqMkCjK5SUFyqrLxiHcsv1LHfFjRnjdZk5RXpVLH1wgc8i5+Pl6J+U9BEhfr/ptjxV1SIv8ICfc7ZXctqMzR97SE+mYfLDEmJ01vjupzVjbBM7Jl1goakxJkYHVB/tG8crq/+0EdfrTuk52dv167MfI19d4VGXdJYT1zdTtFh9n24lZVfpBnrD0uS7ryUUSAAF48i6CKdq82zI99YnSoqrTT17NhZ99r8dopaQS0LG38fr7LiJdRfUSF+Z4qas7/KCpyoUH+F+p+7sKkNPpmHGYakxGlgcqyW787Uj7+s1KBLe5r6gQVQX3l5WTS6W4IGJcfoXz/u0NSVB/S/DUf007ZMPTywtcb3Tqx1V7epKw6ouNSmTvHhLI4KwCEogi7Cudo8H80p1D2frtNb47pUWwgZhqH8otKKwqWioDkz9ey3ozdnr15fEwG+XmeN0pSPzvxmtObMCE6Igwqb2uKTeZjB28uinkkROr7NUM+kCAogwIkaBPnpb6M6aEy3pnry21RtPHhSz363VdPXHNSzo1LUvVnNRvuLSqz6ZMU+SdKES5uzhhcAh6AIstP52jyXb3vsq03afDhH2aeKK01NO5ZXpKKzVqeviUBf71+nnp0ZuSkvbiqN3oT6K9jPu078J8En8wBQ/3WID9c39/TRF2sO6oU527X9aJ5ufHu5ruvSRI8PbaeoUP/z/vyszUeVlV+suPAADU1hcVQAjkERZKdVadkXbPOcW1iqNxbuOefjwX7e1TYOKC9qokJ/3RbsXz9/VXwyDwD1n5eXRWN7NNWQ9rH659zt+nz1Qc1Yd1jztmbo0UFtNK5XYrX//huG9OGy/ZLKFkd1VKc5AKif76xdoKbtm/u1bKTuzSLU6KyCJirEX41C/RTkR/oBAJ6jYbCf/nFdR43ulqCnvk1V6uFcTZy5RV+uOahnrkmpuN+nvOHQ9we8tCMjX4G+Xvpdj6YmRw+gPuFduJ1q2r753gEtHdIaFACA+qJz04b69t5+mrbqgF6cs11bjuTq+reWaXS3eHVvFqGX5+08M9uibOTHYrFo+Z4s7hcF4DCMK9upfAHGc03eskiKo80zAADV8vay6JZeiVr46OW6sWu8JOnLNYf05682VZluXlBs1T2frtOc1HQzQgVQD1EE2am8zbOkKoUQbZ4BAKiZyBB/vXhjJ315dy/5XOD/zMmztspqq64lEQDUDkXQRShv8xwbXnlqXGx4wDnbYwMAgKqsNqn0PAWOISk9p1Cr0rJdFxSAeot7gi4SbZ4BALh4NW04VNP9AOB8KIIcgDbPAABcnJo2HKrpfgBwPkyHAwAApqPhEABXoggCAACmo+EQAFeiCAIAAG6BhkMAXIV7ggAAgNug4RAAV6AIAgAAboWGQwCcjelwAAAAADwKRRAAAAAAj0IRBAAAAMCjUAQBAAAA8CgUQQAAAAA8CkUQAAAAAI9CEQQAAADAo1AEAQAAAPAoFEEAAAAAPApFEAAAAACPQhEEAAAAwKNQBAEAAADwKBRBAAAAADyKj9kBXAzDMCRJubm5JkcilZSUqKCgQLm5ufL19TU7nDqF3NmHvNmHvNmP3NmHvNmHvNmHvNmP3NnHnfJWXhOU1wjnU6eLoLy8PElSQkKCyZEAAAAAcAd5eXkKDw8/7z4Woyalkpuy2Ww6cuSIQkNDZbFYTI0lNzdXCQkJOnjwoMLCwkyNpa4hd/Yhb/Yhb/Yjd/Yhb/Yhb/Yhb/Yjd/Zxp7wZhqG8vDw1btxYXl7nv+unTo8EeXl5KT4+3uwwKgkLCzP9CVBXkTv7kDf7kDf7kTv7kDf7kDf7kDf7kTv7uEveLjQCVI7GCAAAAAA8CkUQAAAAAI9CEeQg/v7+mjhxovz9/c0Opc4hd/Yhb/Yhb/Yjd/Yhb/Yhb/Yhb/Yjd/apq3mr040RAAAAAKC2GAkCAAAA4FEoggAAAAB4FIogAAAAAB6FIggAAACAR6EIOss//vEPde/eXaGhoYqOjtaoUaO0Y8eOSvsUFhbq3nvvVWRkpEJCQnT99dcrIyOj0j4PPPCAunbtKn9/f11yySXnPefu3bsVGhqqBg0aOPhqXMdVedu3b58sFkuVrxUrVjjz8pzGlc83wzD0r3/9S61bt5a/v7+aNGmi5557zlmX5nSuyt2kSZOqfc4FBwc78/KcxpXPublz56pXr14KDQ1VVFSUrr/+eu3bt89JV+Zcrszbl19+qUsuuURBQUFKTEzUiy++6KzLcglH5G7jxo0aO3asEhISFBgYqHbt2um1116rcq6ff/5ZXbp0kb+/v1q2bKkPP/zQ2ZfnNK7KW3p6um666Sa1bt1aXl5eeuihh1xxeU7jqrzNmDFDAwcOVFRUlMLCwtS7d2/NnTvXJdfoDK7K25IlS9S3b19FRkYqMDBQbdu21SuvvOKSa6wORdBZFi1apHvvvVcrVqzQvHnzVFJSokGDBunUqVMV+zz88MOaNWuWpk+frkWLFunIkSO67rrrqhzrjjvu0JgxY857vpKSEo0dO1aXXnqpw6/FlVydt59++knp6ekVX127dnX4NbmCK/P24IMP6r333tO//vUvbd++XTNnzlSPHj2ccl2u4KrcPfroo5Wea+np6UpOTtaNN97otGtzJlflLS0tTddcc42uuOIKbdiwQXPnzlVWVla1x6kLXJW32bNn6+abb9Yf/vAHpaam6s0339Qrr7yi119/3WnX5myOyN3atWsVHR2tTz/9VFu2bNFf//pXPf7445XykpaWpmHDhmnAgAHasGGDHnroId1555119o2pq/JWVFSkqKgoPfnkk+rUqZNLr9EZXJW3xYsXa+DAgfrhhx+0du1aDRgwQCNGjND69etder2O4qq8BQcH67777tPixYu1bds2Pfnkk3ryySf1zjvvuPR6Kxg4p8zMTEOSsWjRIsMwDOPkyZOGr6+vMX369Ip9tm3bZkgyli9fXuXnJ06caHTq1Omcx3/ssceMcePGGVOmTDHCw8MdHb5pnJW3tLQ0Q5Kxfv16Z4VuKmflbevWrYaPj4+xfft2p8VuNme/Vstt2LDBkGQsXrzYYbGbyVl5mz59uuHj42NYrdaKbTNnzjQsFotRXFzs+AtxMWflbezYscYNN9xQadu///1vIz4+3rDZbI69CJNcbO7K/fGPfzQGDBhQ8f1jjz1mtG/fvtI+Y8aMMQYPHuzgKzCHs/J2tv79+xsPPvigQ+M2myvyVi45OdmYPHmyYwI3mSvzdu211xrjxo1zTOC1xEjQeeTk5EiSIiIiJJVVuSUlJbrqqqsq9mnbtq2aNm2q5cuX1+rYCxYs0PTp0/XGG284LmA34cy8SdLIkSMVHR2tfv36aebMmY4J2g04K2+zZs1S8+bN9d133ykpKUnNmjXTnXfeqezsbMdegImc/Zwr995776l169Z1fvS2nLPy1rVrV3l5eWnKlCmyWq3KycnRJ598oquuukq+vr6OvQgTOCtvRUVFCggIqLQtMDBQhw4d0v79+x0QufkclbucnJyKY0jS8uXLKx1DkgYPHnxRr3d34qy81XeuypvNZlNeXl69ya2r8rZ+/XotW7ZM/fv3d1DktUMRdA42m00PPfSQ+vbtq5SUFEnS0aNH5efnV+X+nZiYGB09erTGxz5+/Lhuu+02ffjhhwoLC3Nk2KZzZt5CQkL00ksvafr06fr+++/Vr18/jRo1ql4UQs7M2969e7V//35Nnz5dH3/8sT788EOtXbtWN9xwgyMvwTTOzN3ZCgsLNXXqVE2YMOFiQ3YLzsxbUlKSfvzxRz3xxBPy9/dXgwYNdOjQIX355ZeOvARTODNvgwcP1owZMzR//nzZbDbt3LlTL730kqSyezfqOkflbtmyZfriiy/0+9//vmLb0aNHFRMTU+UYubm5On36tGMvxMWcmbf6zJV5+9e//qX8/HyNHj3aYfGbxRV5i4+Pl7+/v7p166Z7771Xd955p8OvoyZ8TDlrHXDvvfcqNTVVS5Yscfix77rrLt1000267LLLHH5sszkzb40aNdIjjzxS8X337t115MgRvfjiixo5cqTDz+dKzsybzWZTUVGRPv74Y7Vu3VqS9P7776tr167asWOH2rRp4/BzupIzc3e2b775Rnl5eRo/frxTz+Mqzszb0aNHddddd2n8+PEaO3as8vLy9PTTT+uGG27QvHnzZLFYHH5OV3H2/w179uzR8OHDVVJSorCwMD344IOaNGmSvLzq/meWjshdamqqrrnmGk2cOFGDBg1yYHTui7zZx1V5mzZtmiZPnqxvv/1W0dHRdp/LXbgib7/88ovy8/O1YsUK/d///Z9atmypsWPHXkzYdqn7/6o6wX333afvvvtOCxcuVHx8fMX22NhYFRcX6+TJk5X2z8jIUGxsbI2Pv2DBAv3rX/+Sj4+PfHx8NGHCBOXk5MjHx0cffPCBoy7D5Zydt+r07NlTu3fvvqhjmM3ZeYuLi5OPj09FASRJ7dq1kyQdOHDg4oI3mSufc++9956GDx9e5dPmusjZeXvjjTcUHh6uf/7zn+rcubMuu+wyffrpp5o/f75WrlzpqMtwOWfnzWKx6IUXXlB+fr7279+vo0ePVjQwad68uUOuwSyOyN3WrVt15ZVX6ve//72efPLJSo/FxsZW6caXkZGhsLAwBQYGOvZiXMjZeauvXJW3zz//XHfeeae+/PLLKtMx6yJX5S0pKUkdOnTQXXfdpYcffliTJk1y9KXUCEXQWQzD0H333advvvlGCxYsUFJSUqXHu3btKl9fX82fP79i244dO3TgwAH17t27xudZvny5NmzYUPH1zDPPKDQ0VBs2bNC1117rsOtxFVflrTobNmxQXFzcRR3DLK7KW9++fVVaWqo9e/ZUbNu5c6ckKTEx8SKvwhyufs6lpaVp4cKFdX4qnKvyVlBQUGXkwtvbW1LZyGRd4+rnm7e3t5o0aSI/Pz999tln6t27t6Kioi76OszgqNxt2bJFAwYM0Pjx46tt79+7d+9Kx5CkefPmXfT/MWZxVd7qG1fm7bPPPtPtt9+uzz77TMOGDXPOBbmImc+38tkqpjClHYObuueee4zw8HDj559/NtLT0yu+CgoKKvb5wx/+YDRt2tRYsGCBsWbNGqN3795G7969Kx1n165dxvr16427777baN26tbF+/Xpj/fr1RlFRUbXnrevd4VyVtw8//NCYNm2asW3bNmPbtm3Gc889Z3h5eRkffPCBS6/XUVyVN6vVanTp0sW47LLLjHXr1hlr1qwxevbsaQwcONCl1+tIrn6tPvnkk0bjxo2N0tJSl1yfs7gqb/PnzzcsFosxefJkY+fOncbatWuNwYMHG4mJiZXOVVe4Km/Hjh0z3nrrLWPbtm3G+vXrjQceeMAICAgwVq5c6dLrdSRH5G7z5s1GVFSUMW7cuErHyMzMrNhn7969RlBQkPHnP//Z2LZtm/HGG28Y3t7expw5c1x6vY7iqrwZhlHxPOzatatx0003GevXrze2bNnismt1JFflberUqYaPj4/xxhtvVNrn5MmTLr1eR3FV3l5//XVj5syZxs6dO42dO3ca7733nhEaGmr89a9/den1lqMIOoukar+mTJlSsc/p06eNP/7xj0bDhg2NoKAg49prrzXS09MrHad///7VHictLa3a89b1IshVefvwww+Ndu3aGUFBQUZYWJjRo0ePSu0a6xpXPt8OHz5sXHfddUZISIgRExNj3Hbbbcbx48dddKWO58rcWa1WIz4+3njiiSdcdHXO48q8ffbZZ0bnzp2N4OBgIyoqyhg5cqSxbds2F12pY7kqb8eOHTN69eplBAcHG0FBQcaVV15prFixwoVX6niOyN3EiROrPUZiYmKlcy1cuNC45JJLDD8/P6N58+aVzlHXuDJvNdmnrnBV3s71Wh4/frzrLtaBXJW3f//730b79u0r3sd17tzZePPNNystp+BKFsMwDAEAAACAh+CeIAAAAAAehSIIAAAAgEehCAIAAADgUSiCAAAAAHgUiiAAAAAAHoUiCAAAAIBHoQgCAAAA4FEoggAAAAB4FIogAAAAAB6FIggA4DYMw9BVV12lwYMHV3nszTffVIMGDXTo0CETIgMA1CcUQQAAt2GxWDRlyhStXLlS//3vfyu2p6Wl6bHHHtN//vMfxcfHO/ScJSUlDj0eAMD9UQQBANxKQkKCXnvtNT366KNKS0uTYRiaMGGCBg0apM6dO2vo0KEKCQlRTEyMbrnlFmVlZVX87Jw5c9SvXz81aNBAkZGRGj58uPbs2VPx+L59+2SxWPTFF1+of//+CggI0NSpU824TACAiSyGYRhmBwEAwG+NGjVKOTk5uu666/Tss89qy5Ytat++ve68807deuutOn36tP7yl7+otLRUCxYskCR9/fXXslgs6tixo/Lz8/X0009r37592rBhg7y8vLRv3z4lJSWpWbNmeumll9S5c2cFBAQoLi7O5KsFALgSRRAAwC1lZmaqffv2ys7O1tdff63U1FT98ssvmjt3bsU+hw4dUkJCgnbs2KHWrVtXOUZWVpaioqK0efNmpaSkVBRBr776qh588EFXXg4AwI0wHQ4A4Jaio6N19913q127dho1apQ2btyohQsXKiQkpOKrbdu2klQx5W3Xrl0aO3asmjdvrrCwMDVr1kySdODAgUrH7tatm0uvBQDgXnzMDgAAgHPx8fGRj0/Zf1X5+fkaMWKEXnjhhSr7lU9nGzFihBITE/Xuu++qcePGstlsSklJUXFxcaX9g4ODnR88AMBtUQQBAOqELl266Ouvv1azZs0qCqOzHT9+XDt27NC7776rSy+9VJK0ZMkSV4cJAKgDmA4HAKgT7r33XmVnZ2vs2LFavXq19uzZo7lz5+r222+X1WpVw4YNFRkZqXfeeUe7d+/WggUL9Mgjj5gdNgDADVEEAQDqhMaNG2vp0qWyWq0aNGiQOnTooIceekgNGjSQl5eXvLy89Pnnn2vt2rVKSUnRww8/rBdffNHssAEAbojucAAAAAA8CiNBAAAAADwKRRAAAAAAj0IRBAAAAMCjUAQBAAAA8CgUQQAAAAA8CkUQAAAAAI9CEQQAAADAo1AEAQAAAPAoFEEAAAAAPApFEAAAAACPQhEEAAAAwKP8P6KQ14ErFH3sAAAAAElFTkSuQmCC", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0EAAAIjCAYAAADFthA8AAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAdE5JREFUeJzt3Xd4VGX6xvF7Jpn0QhLSgBBCJwmg9CYIUqQKFlyxYF3XtZfd/bmrArquZa3r2laxgxVUQAGRJr3XQKgJoQRCEtJISJvz+yMkEgEhMMmZyXw/15VLc+ZkzjPkJcyd9z3PazEMwxAAAAAAuAmr2QUAAAAAQF0iBAEAAABwK4QgAAAAAG6FEAQAAADArRCCAAAAALgVQhAAAAAAt0IIAgAAAOBWCEEAAAAA3AohCAAAAIBbIQQBANzS5Zdfrssvv9zsMqp8+umnatu2rWw2mxo0aCCpdmqcOHGiLBaLQ58TAFwNIQgAHOytt96SxWJR9+7dzS7FaaxYsUJWq1WPP/74GR9/4YUXZLFY9MMPP9RxZY5jsVh03333XdDXJicn69Zbb1WLFi303nvv6X//+99F1VJYWKiJEydq0aJFF/U8AFBfEYIAwMGmTJmiZs2aafXq1dq9e7fZ5TiFnj176u6779bLL7+spKSkao/t27dPTz/9tK677joNHz7cpArNtWjRItntdr3++uu69dZbNXbs2It6vsLCQk2aNOmMIeiJJ55QUVHRRT0/ALg6QhAAOFBKSoqWL1+uV155ReHh4ZoyZUqd12C323XixIk6v+65PP/882rYsKHuvvtuGYZRdfz++++XzWbT66+/Xid1FBYW1sl1aiIjI0OSqpbB1SZPT0/5+PjU+nUAwJkRggDAgaZMmaKQkBANHz5c1157bbUQVFpaqtDQUN12222nfV1eXp58fHz02GOPVR0rLi7WhAkT1LJlS3l7eysmJkZ//etfVVxcXO1rK5dhTZkyRQkJCfL29tacOXMkSS+99JJ69eqlsLAw+fr6qnPnzvrmm29Ou35RUZEeeOABNWzYUIGBgRo1apQOHjwoi8WiiRMnVjv34MGDuv322xUZGSlvb28lJCTogw8+OOefTXBwsF5//XUtW7ZM77//viTp22+/1cyZM/X8888rOjpadrtdr732mhISEuTj46PIyEjdfffdOnbsWLXn+v777zV8+HA1atRI3t7eatGihZ555hmVl5dXO+/yyy9XYmKi1q1bp759+8rPz09///vfT6utoKBA/v7+evDBB0977MCBA/Lw8NBzzz13ztd4qkWLFsliseirr77Ss88+qyZNmsjHx0dXXHFFtRnCZs2aacKECZKk8PDwM/6ZVyopKdFTTz2lzp07Kzg4WP7+/rrsssu0cOHCqnNSU1MVHh4uSZo0aZIsFku15zzTPUFlZWV65pln1KJFC3l7e6tZs2b6+9//ftpYa9asmUaMGKGlS5eqW7du8vHxUfPmzfXJJ5/U6M8GAExnAAAcpm3btsYdd9xhGIZh/PLLL4YkY/Xq1VWP33777UaDBg2M4uLial/38ccfG5KMNWvWGIZhGOXl5cbgwYMNPz8/46GHHjLeffdd47777jM8PT2Nq666qtrXSjLatWtnhIeHG5MmTTLefPNNY8OGDYZhGEaTJk2MP//5z8Z///tf45VXXjG6detmSDJmzZpV7TnGjh1rSDJuvvlm48033zTGjh1rdOzY0ZBkTJgwoeq8w4cPG02aNDFiYmKMp59+2nj77beNUaNGGZKMV1999bz+jIYPH26EhIQYe/bsMWJiYoxevXoZdrvdMAzDuPPOOw1PT0/jrrvuMt555x3jb3/7m+Hv72907drVKCkpqXqO0aNHG2PHjjX+/e9/G2+//bZx3XXXGZKMxx57rNq1+vXrZ0RFRRnh4eHG/fffb7z77rvGd999V/VYv379qs698cYbjcjISKOsrKzac7z44ouGxWIx9u3b97uvS5Jx7733Vn2+cOFCQ5Jx6aWXGp07dzZeffVVY+LEiYafn5/RrVu3qvO+/fZbY8yYMYYk4+233zY+/fRTY9OmTWes8ejRo0Z0dLTxyCOPGG+//bbx4osvGm3atDFsNlvV97ygoMB4++23DUnGmDFjjE8//bTac06YMMH47T//48ePNyQZ1157rfHmm28at9xyiyHJGD16dLXzYmNjjTZt2hiRkZHG3//+d+O///2v0alTJ8NisRhbt2793T8fAHAmhCAAcJC1a9cakox58+YZhmEYdrvdaNKkifHggw9WnTN37lxDkjFz5sxqXzts2DCjefPmVZ9/+umnhtVqNZYsWVLtvHfeeceQZCxbtqzqmCTDarUaSUlJp9VUWFhY7fOSkhIjMTHRGDBgQNWxdevWGZKMhx56qNq5t95662kh6I477jCio6ONzMzMauf+4Q9/MIKDg0+73pmkpqYa/v7+RmhoqGGz2YwtW7YYhmEYS5YsMSQZU6ZMqXb+nDlzTjt+puvcfffdhp+fn3HixImqY/369TMkGe+8885p5/82YFR+b2bPnl3tvA4dOlQ772zOFoLatWtXLfS+/vrrhqSq120YvwaTo0eP/m6NZWVlpwXoY8eOGZGRkcbtt99edezo0aOnfe9+e61KGzduNCQZd955Z7XzHnvsMUOSsWDBgqpjsbGxhiTjl19+qTqWkZFheHt7G48++ujZ/mgAwOmwHA4AHGTKlCmKjIxU//79JVUsU7v++uv1xRdfVC3TGjBggBo2bKgvv/yy6uuOHTumefPm6frrr6869vXXX6tdu3Zq27atMjMzqz4GDBggSdWWP0lSv379FB8ff1pNvr6+1a6Tm5uryy67TOvXr686Xrl07s9//nO1r73//vurfW4YhqZNm6aRI0fKMIxqdQ0ZMkS5ubnVnvdsYmNjNWHCBGVnZ+uRRx5RYmJi1WsODg7WoEGDqj13586dFRAQUO01n/q68vPzlZmZqcsuu0yFhYVKTk6udj1vb+8zLkH8rYEDB6pRo0bVljBu3bpVmzdv1k033XTOrz+b2267TV5eXlWfX3bZZZKkvXv31vi5PDw8qp7LbrcrOztbZWVl6tKly3n92Z/Jjz/+KEl65JFHqh1/9NFHJem0jn3x8fFVr0GqWMLXpk2bC3o9AGAWT7MLAID6oLy8XF988YX69++vlJSUquPdu3fXyy+/rPnz52vw4MHy9PTUNddco6lTp6q4uFje3t6aPn26SktLq4WgXbt2afv27VX3dvxW5Y30leLi4s543qxZs/TPf/5TGzdurHZ/x6n3hOzbt09Wq/W052jZsmW1z48ePaqcnBz973//O2sL59/WdTZdu3aVJHXp0qXq2K5du5Sbm6uIiIhzPndSUpKeeOIJLViwQHl5edXOy83NrfZ548aNq4WQs7Farbrxxhv19ttvq7CwUH5+fpoyZYp8fHx03XXXndfrOpOmTZtW+zwkJESSTrvP6Xx9/PHHevnll5WcnKzS0tKq42cbA+dS+f3/7fc7KipKDRo00L59+6od/+3rkSpe04W+HgAwAyEIABxgwYIFSk9P1xdffKEvvvjitMenTJmiwYMHS5L+8Ic/6N1339Xs2bM1evRoffXVV2rbtq06duxYdb7dblf79u31yiuvnPF6MTEx1T4/dWak0pIlSzRq1Cj17dtXb731lqKjo2Wz2fThhx9q6tSpNX6NdrtdknTTTTdp/PjxZzynQ4cONX7eU58/IiLirB31KgNhTk6O+vXrp6CgID399NNq0aKFfHx8tH79ev3tb3+rqrPSmf5szuaWW27Rv//9b3333Xe64YYbNHXqVI0YMULBwcEX/Lo8PDzOeNw4pUPe+frss8906623avTo0frLX/6iiIiIqqYNe/bsueAaJZ33BqqOfD0AYBZCEAA4wJQpUxQREaE333zztMemT5+ub7/9Vu+88458fX3Vt29fRUdH68svv1SfPn20YMEC/eMf/6j2NS1atNCmTZt0xRVXnPeb09+aNm2afHx8NHfuXHl7e1cd//DDD6udFxsbK7vdrpSUFLVq1arq+G/3OAoPD1dgYKDKy8s1cODAC6rp97Ro0UI///yzevfu/bvBZdGiRcrKytL06dPVt2/fquOnzsBdqMTERF166aWaMmWKmjRporS0NL3xxhsX/byO8s0336h58+aaPn16tXFR2V2uUk3GTOX3f9euXWrXrl3V8SNHjignJ0exsbEXXzgAOBnuCQKAi1RUVKTp06drxIgRuvbaa0/7uO+++5Sfn68ZM2ZIqlh2de2112rmzJn69NNPVVZWVm0pnCSNHTtWBw8e1HvvvXfG6x0/fvycdXl4eMhisVRrG52amqrvvvuu2nlDhgyRJL311lvVjv/2zb+Hh4euueYaTZs2TVu3bj3tekePHj1nTb9n7NixKi8v1zPPPHPaY2VlZcrJyamqQ6o+81BSUnJa/Rfq5ptv1k8//aTXXntNYWFhGjp0qEOe1xHO9NpXrVqlFStWVDvPz89Pkqr+zH7PsGHDJEmvvfZateOVs5DuuoEtgPqNmSAAuEgzZsxQfn6+Ro0adcbHe/ToUbVxamXYuf766/XGG29owoQJat++fbXfwEsVb8S/+uor/elPf9LChQvVu3dvlZeXKzk5WV999ZXmzp1b7X6aMxk+fLheeeUVXXnllRo3bpwyMjL05ptvqmXLltq8eXPVeZ07d9Y111yj1157TVlZWerRo4cWL16snTt3Sqo+q/D8889r4cKF6t69u+666y7Fx8crOztb69ev188//6zs7OwL+jOUKpo73H333Xruuee0ceNGDR48WDabTbt27dLXX3+t119/Xddee6169eqlkJAQjR8/Xg888IAsFos+/fRThy3HGjdunP7617/q22+/1T333CObzeaQ53WEESNGaPr06RozZoyGDx+ulJQUvfPOO4qPj1dBQUHVeb6+voqPj9eXX36p1q1bKzQ0VImJiVVNKE7VsWNHjR8/Xv/73/+qlhquXr1aH3/8sUaPHl3V6AMA6hNCEABcpMqb5wcNGnTGx61Wq4YPH64pU6YoKytLYWFh6tWrl2JiYrR///7TZoEqv+a7777Tq6++qk8++UTffvut/Pz81Lx5cz344INq3br1OesaMGCAJk+erOeff14PPfSQ4uLi9MILLyg1NbVaCJKkTz75RFFRUfr888/17bffauDAgfryyy/Vpk0b+fj4VJ0XGRmp1atX6+mnn9b06dP11ltvKSwsTAkJCXrhhRdq+Cd3unfeeUedO3fWu+++q7///e/y9PRUs2bNdNNNN6l3796SpLCwMM2aNUuPPvqonnjiCYWEhOimm27SFVdcUTWrdTEiIyM1ePBg/fjjj7r55psv+vkc6dZbb9Xhw4f17rvvau7cuYqPj9dnn32mr7/+WosWLap27vvvv6/7779fDz/8sEpKSjRhwoQzhqDKc5s3b66PPvpI3377raKiovT444+ftswOAOoLi8GdjACAM9i4caMuvfRSffbZZ7rxxhvNLqdOjRkzRlu2bDntvigAQP3APUEAABUVFZ127LXXXpPVaq3WfMAdpKen64cffnC6WSAAgOOwHA4AoBdffFHr1q1T//795enpqdmzZ2v27Nn64x//eFo77voqJSVFy5Yt0/vvvy+bzaa7777b7JIAALWEEAQAUK9evTRv3jw988wzKigoUNOmTTVx4sTTWnfXZ4sXL9Ztt92mpk2b6uOPP1ZUVJTZJQEAagn3BAEAAABwK9wTBAAAAMCtEIIAAAAAuBWXvifIbrfr0KFDCgwMrLaZHwAAAAD3YhiG8vPz1ahRI1mtvz/X49Ih6NChQ27TtQgAAADAue3fv19NmjT53XNcOgQFBgZKqnihQUFBptZSWlqqn376SYMHD5bNZjO1FrgHxhzqGmMOdYnxhrrGmHN9eXl5iomJqcoIv8elQ1DlErigoCCnCEF+fn4KCgriLw7qBGMOdY0xh7rEeENdY8zVH+dzmwyNEQAAAAC4FUIQAAAAALdCCAIAAADgVghBAAAAANwKIQgAAACAWyEEAQAAAHArhCAAAAAAboUQBAAAAMCtEIIAAAAAuBVCEAAAAAC3QggCAAAA4FYIQQAAAADcCiEIAAAAgFshBAEAAABwK4QgAAAAAG6FEAQAAADArRCCAAAA4NYMw9CGtByVlJtdCeoKIQgAAABubebmdI19b7U+2GmVYRhml4M6QAgCAACAW/tuw0FJ0vYcq+YkHTG5GtQFQhAAAADcVv6JUi3dlVn1+b9m79Dx4jITK0JdIAQBAADAbS1IzlBJuV2xoX4K8zZ0OK9Y/1mwy+yyUMsIQQAAAHBbc7YeliQNS4zU1XF2SdLkJSnanZFvZlmoZYQgAAAAuKWiknIt2nFUkjQ4PlKJIYauaBuuMruhJ79LoklCPUYIAgAAgFtavPOoikrL1biBrxIaBUqS/jGsjbw9rVqxN0szN6ebXCFqCyEIAAAAbmluUsVSuCsTo2SxWCRJMSF+urd/S0nSP2dtU/6JUtPqQ+0hBAEAAMDtlJTZ9fP2inbYQxOjqj32x77N1SzMTxn5xXr9Z5ok1EeEIAAAALid5XsylX+iTOGB3urUNKTaYz42D00clSBJ+nB5qnYcpklCfUMIAgAAgNupXAo3JCFSVqvltMcvbxOhKxOiVG439OT3W2mSUM+YHoIOHjyom266SWFhYfL19VX79u21du1as8sCAABAPVVuN/RTUsVSuCsTos963pMj4+Vr89DqlGx9t/FgXZWHOmBqCDp27Jh69+4tm82m2bNna9u2bXr55ZcVEhJy7i8GAAAALsCa1GxlHS9RAz+bujcPPet5jRv46v4rKpokPPtDsnKLaJJQX3iaefEXXnhBMTEx+vDDD6uOxcXFmVgRAAAA6rvKDVIHtouUzeP35wTu7NNc36w7oL1Hj+vVeTur7hWCazM1BM2YMUNDhgzRddddp8WLF6tx48b685//rLvuuuuM5xcXF6u4uLjq87y8PElSaWmpSkvNTeaV1ze7DrgPxhzqGmMOdYnxhtpitxuavbVi/59B7cJPG2u/HXMWSU8Nb6tbP1qnT1ak6upLotUuOrBOa8b5qcnPC4th4l1ePj4+kqRHHnlE1113ndasWaMHH3xQ77zzjsaPH3/a+RMnTtSkSZNOOz516lT5+fnVer0AAABwban50qtbPeVtNfRs13LZzvPmkI92WrUhy6q4QEMPJJTrDL0UYLLCwkKNGzdOubm5CgoK+t1zTQ1BXl5e6tKli5YvX1517IEHHtCaNWu0YsWK084/00xQTEyMMjMzz/lCa1tpaanmzZunQYMGyWazmVoL3ANjDnWNMYe6xHhDbXlx7k69tzRVwxOj9Nr1HaqOn2vMpeee0JX/WabCknI9PyZB13RqXJdl4zzk5eWpYcOG5xWCTF0OFx0drfj4+GrH2rVrp2nTpp3xfG9vb3l7e5923GazOc0PSGeqBe6BMYe6xphDXWK8wZEMw9BP2zMkScM6NDrj2DrbmGva0KaHBrbSv35M1r9/2qWh7Rsr2I+x6Uxq8rPC1O5wvXv31o4dO6od27lzp2JjY02qCAAAAPVV8uF87csqlLenVZe3Ca/x19/WO06tIgKUdbxEL/2049xfAKdlagh6+OGHtXLlSv3rX//S7t27NXXqVP3vf//Tvffea2ZZAAAAqIdmn+wK17d1uPy9a74gyuZh1dNXJUqSPlu1T1sO5Dq0PtQdU0NQ165d9e233+rzzz9XYmKinnnmGb322mu68cYbzSwLAAAA9dDckyHoyoSoC36Oni3CdNUljWQY0hPfb5Xdbtrt9bgIpt4TJEkjRozQiBEjzC4DAAAA9djeowXacSRfnlaLBraLvKjn+sewdpq/PUOb9ufoq7X79YduTR1UJeqKqTNBAAAAQF2Yk1QxC9SzRdhFNzSICPLRw4NaS5JemJOsY8dLLro+1C1CEAAAAOq9yqVwQxOjHfJ843vGqm1UoI4VlurFuTRJcDWEIAAAANRrB3OKtOlAriwWaVD8xS2Fq+R5SpOEL9akaeP+HIc8L+oGIQgAAAD1WuUsUNdmoQoPPH3PyQvVLS5UV3dqLMOQnvxuq8ppkuAyCEEAAACo1+Y4oCvc2Tw+tJ0CfTy15WCupq5Oc/jzo3YQggAAAFBvHc0v1pp92ZKkKxMdH4LCA7312OA2kqR/z0lWVkGxw68BxyMEAQAAoN76adthGYbUsUmwGjXwrZVr3Ni9qeKjg5R3okwvzEmulWvAsQhBAAAAqLcql8INqYVZoEqeHlY9M7qiScJXaw9o3cmZJzgvQhAAAADqpdzCUq3YkyWpdu4HOlXn2BCN7dJEkvTkd0kqK7fX6vVwcQhBAAAAqJd+3n5EZXZDbSID1Tw8oNav97cr2yrY16Zt6Xn6bOW+Wr8eLhwhCAAAAPXSnKSTXeFqcSncqcICvPWXIRVNEl7+aaeO5tMkwVkRggAAAFDvHC8u0y87j0qquxAkSTd0a6oOTYKVX1ym52Zvr7PromYIQQAAAKh3Fu04quIyu5qF+altVGCdXdfDatEzVyXKYpGmrz+o1Sk0SXBGhCAAAADUO7O3pkuq6ApnsVjq9NodYxroD12bSpKe/G6rSmmS4HQIQQAAAKhXTpSWa2FyhiRpaGK0KTX8dUgbhfjZtONIvj5enmpKDTg7QhAAAADqlaW7MnW8pFzRwT7q0DjYlBpC/L30tyvbSpJe+3mXjuSdMKUOnBkhCAAAAPVKZVe4IQlRslrrdincqcZ2idElMQ1UUFymZ3+gSYIzIQQBAACg3igtt2vetiOS6rYr3JlYrRb9c3RFk4QZmw5p+Z5MU+vBrwhBAAAAqDdW7c1WblGpwvy91LVZqNnlKLFxsG7qHitJeur7JJokOAlCEAAAAOqNyq5wgxMi5WHiUrhTPTa4jcL8vbQ7o0AfLE0xuxyIEAQAAIB6otxuaG5SxVK4IQnmLoU7VbCfTf83tKJJwuvzdyk9t8jkikAIAgAAQL2wPu2YMguKFejjqV4tGppdTjXXdGqiLrEhKiwp1z9n0STBbIQgAAAA1AtztlZ0hRvYLlJens71NtdqtejpqxJltUg/bEnXkl1HzS7JrTnX6AAAAAAugGEYVSHI7K5wZxPfKEi39GwmSZrwfZKKy8rNLciNEYIAAADg8rYezNPBnCL52jzUt1W42eWc1SODW6thgLf2Zh7X+0tokmAWQhAAAABc3pykiq5w/duGy9fLw+Rqzi7Ix6Z/DK9okvDGgl06mEOTBDMQggAAAODSDMPQ7JNL4ZypK9zZjL6ksbrFhepEqV1Pz0wyuxy3RAgCAACAS9udUaC9R4/Ly8OqAW0jzC7nnCwWi565KlEeVovmJh3Rwh0ZZpfkdghBAAAAcGmVs0B9WjVUoI/N5GrOT5uoQN3Wq5kkaeKMJJ0opUlCXSIEAQAAwKU5e1e4s3loUGtFBnlrX1ah/vfLXrPLcSuEIAAAAListKxCbUvPk4fVooHtIs0up0YCvD31j+HxkqQ3F+7W/uxCkytyH4QgAAAAuKzKrnDd40IV6u9lcjU1N7JDtHq1CFNxmV2TaJJQZwhBAAAAcFmV9wMNdbGlcJUsFouevipBNg+Lft6eoZ+3HTG7JLdACAIAAIBLOpx7QhvSciRJg12gNfbZtIwI1B19mkuSJs2iSUJdIAQBAADAJc1NqpgF6hwbosggH5OruTj3D2ip6GAf7c8u0luL9phdTr1HCAIAAIBLquoK58KzQJX8vT315IiKJgnvLN6j1MzjJldUvxGCAAAA4HKyj5doVUqWJNdrjX02QxOjdFmrhiops2vizCQZhmF2SfUWIQgAAAAuZ962w7IbUkKjIMWE+pldjkNYLBZNGpUgLw+rFu04qp9oklBrCEEAAABwOXNcvCvc2TQPD9Af+1Y0SXh65jYVlpSZXFH9RAgCAACAS8k7UaqluzMl1Z+lcKe6t39LNW7gq4M5RXpz4W6zy6mXCEEAAABwKQuTM1RabqhlRIBaRgSaXY7D+Xp5aMLIiiYJ//tlr/YcLTC5ovqHEAQAAACXMntL/ekKdzaD4iPVv024SssNTZxBkwRHIwQBAADAZRSVlGvRzgxJ9XMpXCWLxaKJoxLk5WnVkl2Z+vFk8INjEIIAAADgMhbvzNCJUruahPgqoVGQ2eXUqtgwf93Tr4Uk6ZlZ23S8mCYJjkIIAgAAgMs4dYNUi8VicjW1757LWygm1FeH807oPwt2mV1OvUEIAgAAgEsoLivX/O0VS+GGtq+/S+FO5WPz0KRRCZKkyUtStOtIvskV1Q+EIAAAALiE5XuylF9cpohAb10aE2J2OXVmQNtIDWwXqTK7oae+p0mCIxCCAAAA4BLmnlwKNyQhSlZr/V8Kd6oJI+Pl7WnVir1ZmrHpkNnluDxCEAAAAJxeWbldP207Iql+d4U7m5hQP93Xv6Uk6dkftiv/RKnJFbk2QhAAAACc3prUY8o+XqIGfjZ1jws1uxxT3NW3uZqF+Skjv1iv/UyThItBCAIAAIDTm7M1XZI0qF2kPD3c8y2sj81DE082SfhoeaqSD+eZXJHrcs8RBAAAAJdhtxuam1SxFM5dusKdzeVtInRlQpTK7Yae+o4mCReKEAQAAACntvFAjg7nnVCAt6d6t2xodjmme3JkvHxtHlqdmq1vNxw0uxyXRAgCAACAU6vsCjegbYS8PT1MrsZ8jRv46v4rKpok/OvH7cotoklCTRGCAAAA4LQMw9DskyHIHbvCnc2dfZqrebi/MgtK9Oq8nWaX43IIQQAAAHBa29PzlZZdKG9Pqy5vE252OU7Dy9Oqp0clSpI+WZGqpEO5JlfkWghBAAAAcFqVXeH6tQ6Xn5enydU4lz6tGmp4h2jZDemp75Nkt9Mk4XwRggAAAOC05iSxFO73PDk8Xn5eHlq375i+WX/A7HJcBiEIAAAATmnP0QLtPFIgT6tFV7SLNLscpxQV7KOHBraSJD0/O1m5hTRJOB+EIAAAADilOScbIvRq2VDBvjaTq3Fet/WOU6uIAGUfL9G/f0o2uxyXQAgCAACAU5p7cincUJbC/S6bh1VPX1XRJGHKqjRtOUCThHMhBAEAAMDpHDhWqM0HcmW1SIPiWQp3Lj1bhOmqSxrJMKQnvt9Kk4RzIAQBAADA6cxNOiJJ6tosVA0DvE2uxjX8Y1g7BXh7atP+HH25dr/Z5Tg1QhAAAACcTmVrbLrCnb+IIB89PKi1JOmFOck6drzE5IqcFyEIAAAATiUj/4TW7jsmSRqSQAiqifE9Y9U2KlA5haV6cS5NEs6GEAQAAACn8lPSERmG1DGmgRo18DW7HJfieUqThC/W7NeGtGMmV+ScCEEAAABwKnSFuzjd4kJ1dafGMgzpye+3qpwmCachBAEAAMBp5BSWaMWeLEnSlSyFu2CPD22nQB9PbT2Yp6mr08wux+kQggAAAOA0ft6eoTK7obZRgWrW0N/sclxWeKC3HhvcRpL07znJyiwoNrki50IIAgAAgNOgK5zj3NQjVgmNgpR3okwvzKZJwqkIQQAAAHAKBcVl+mVXpiRCkCN4WC1VTRK+XndA6/Zlm1yR8yAEAQAAwCks2pGhkjK74hr6q01koNnl1AudY0N0fZcYSdIT3yWprNxuckXOgRAEAAAApzB7a0VXuCEJUbJYLCZXU3/89co2Cva1aXt6nj5buc/scpwCIQgAAACmO1FaroXJGZJoje1oYQHe+suQiiYJL/+0Uxn5J0yuyHyEIAAAAJhuya5MFZaUq1Gwjzo0CTa7nHrnhm5N1aFJsPKLy/T8jzRJIAQBAADAdHMql8IlshSuNnhYLXrmqkRZLNL0DQe1am+W2SWZihAEAAAAU5WW2/Xz9iOS2CC1NnWMaaAbujWVJD31fZJK3bhJAiEIAAAAplq5N0u5RaVqGOClLs1CzS6nXvvL4DYK8bNpx5F8fbw81exyTEMIAgAAgKkqu8INio+Sh5WlcLUpxN9L/ze0rSTptZ936UieezZJIAQBAADANOV2Qz8lVSyFoytc3biuc4wuiWmgguIyPfvDdrPLMQUhCAAAAKZZt++YMguKFeTjqR7Nw8wuxy1YrRb9c3SirBZpxqZDWr470+yS6hwhCAAAAKap7Ao3sF2kvDx5a1pXEhsH66YesZKkp2YkqaTMvZokMNIAAABgCsMwNDepIgRdyVK4OvfooDYK8/fS7owCfbgsxexy6pSpIWjixImyWCzVPtq2bWtmSQAAAKgjWw7m6mBOkfy8PNS3dbjZ5bidYD+bHh/WTpL0+vxdSs8tMrmiumP6TFBCQoLS09OrPpYuXWp2SQAAAKgDlUvh+reJkI/Nw+Rq3NPVlzZWl9gQFZaU65+z3KdJgukhyNPTU1FRUVUfDRs2NLskAAAA1DLDMKpC0BCWwpnGarXo6asqmiT8sCVdv+w8anZJdcLT7AJ27dqlRo0aycfHRz179tRzzz2npk2bnvHc4uJiFRcXV32el5cnSSotLVVpaWmd1Hs2ldc3uw64D8Yc6hpjDnWJ8Vb/7TpSoL2Zx2XzsOiyFiGmf6/decy1CvfVzT2a6uMVaZrw/VbNvK+XvF2wSUVNvncWwzCMWqzld82ePVsFBQVq06aN0tPTNWnSJB08eFBbt25VYGDgaedPnDhRkyZNOu341KlT5efnVxclAwAAwAHm7Ldo9gEPJYTY9ce27tWZzBkVlUn/2uihvFKLhseUa3AT0yLCBSssLNS4ceOUm5uroKCg3z3X1BD0Wzk5OYqNjdUrr7yiO+6447THzzQTFBMTo8zMzHO+0NpWWlqqefPmadCgQbLZbKbWAvfAmENdY8yhLjHe6r+Rb65Q8uF8PT8mQdd0amx2OYw5Sd9vStdj32yRj82qOQ/0VuMGvmaXVCN5eXlq2LDheYUg05fDnapBgwZq3bq1du/efcbHvb295e3tfdpxm83mNIPVmWqBe2DMoa4x5lCXGG/1076s40o+nC8Pq0VDEhs51ffYncfcNZ1j9PW6g1qVkq3n5uzUuzd3MbukGqnJ982pFvsVFBRoz549io6ONrsUAAAA1JLKhgg9m4cpxN/L5GpQyWKx6JnRifKwWjQ36YgW7sgwu6RaY2oIeuyxx7R48WKlpqZq+fLlGjNmjDw8PHTDDTeYWRYAAABq0Wy6wjmt1pGBur13M0nSxBlJOlFabm5BtcTUEHTgwAHdcMMNatOmjcaOHauwsDCtXLlS4eFslgUAAFAfpecWaeP+HFks0pD4SLPLwRk8OLC1IoO8tS+rUO8u3mt2ObXC1HuCvvjiCzMvDwAAgDo29+QsUOemIYoI8jG5GpxJgLennhger/s/36C3Fu3WmEsbq2lY/erE7FT3BAEAAKB+m5NUEYKuZCmcUxvRIVq9WoSpuMyuSTOTzC7H4QhBAAAAqBNZBcVanZItSRqSQAhyZhaLRU9flSibh0XzkzP087YjZpfkUIQgAAAA1Il5247IbkiJjYMUE1q/llfVRy0jAnRHn+aSpIkz61eTBEIQAAAA6kTlUrihiWyH4iruH9BS0cE+OnCsSG8tPPNenq6IEAQAAIBal1tUqmW7MyWxFM6V+Ht76qkR8ZKkdxbvVWrmcZMrcgxCEAAAAGrdwuQMlZYbahURoJYRAWaXgxq4MjFKl7VqqJJyuybMSJJhGGaXdNEIQQAAAKh1s7emS6IrnCuqbJLg5WHV4p1HNTfJ9ZskEIIAAABQqwpLyrR451FJhCBXFdfQX3/sW9Ek4ZlZ21RYUmZyRReHEAQAAIBatXjHUZ0otSsm1Ffx0UFml4MLdG//lmrcwFcHc4r03wWu3SSBEAQAAIBadWpXOIvFYnI1uFC+Xh6aMLKiScJ7S/Zqz9ECkyu6cIQgAAAA1JrisnIt2J4hia5w9cGg+Ej1bxOu0nJDE7533SYJhCAAAADUmuW7s5RfXKbIIG9dGtPA7HJwkSwWiyaOSpCXp1VLd2fqxy2HzS7pghCCAAAAUGsqu8INSYiS1cpSuPogNsxf9/RrIamiSUJBses1SSAEAQAAoFaUlds1b1tFO+UrWQpXr9xzeQs1DfXT4bwTemP+LrPLqTFCEAAAAGrF6tRsHSssVYifTd3iQs0uBw7kY/PQxFEVTRImL01RauZxkyuqGU+zCwAAAED9NGdrxf0ig+Ij5enB797rmwFtI3VDtxh1bRaq2DA/s8upEUIQAAAAHM5uNzT3lNbYqJ+eu7qD2SVcECI5AAAAHG7D/hwdyStWoLenerUMM7scoBpCEAAAAByuchZoQLsIeXt6mFwNUB0hCAAAAA5lGEZVa2y6wsEZEYIAAADgUNvS87Q/u0g+Nqv6tQk3uxzgNIQgAAAAOFRlV7h+rcPl50UfLjgfQhAAAAAcqjIE0RUOzooQBAAAAIfZnVGgXRkFsnlY1L9thNnlAGdECAIAAIDDVHaF692yoYJ9bSZXA5wZIQgAAAAOQ1c4uAJCEAAAABxif3ahth7Mk9UiDYqPNLsc4KwIQQAAAHCIyqVw3eJCFRbgbXI1wNkRggAAAOAQlV3hWAoHZ0cIAgAAwEXLyDuhdWnHJElDEglBcG6EIAAAAFy0uduOyDCkS2IaKDrY1+xygN9FCAIAAMBFm1u1QSqzQHB+hCAAAABclGPHS7Rib5Yk6UpCEFwAIQgAAAAX5eftR1RuN9QuOkixYf5mlwOcEyEIAAAAF4WucHA1hCAAAABcsILiMi3ZlSlJGtqeEATXQAgCAADABVuQnKGScruaN/RXq4gAs8sBzgshCAAAABessivckMQoWSwWk6sBzo/nhXxRTk6OVq9erYyMDNnt9mqP3XLLLQ4pDAAAAM7tRGm5Fu7IkERrbLiWGoegmTNn6sYbb1RBQYGCgoKqJX6LxUIIAgAAcBO/7DyqwpJyNW7gq/aNg80uBzhvNV4O9+ijj+r2229XQUGBcnJydOzYsaqP7Ozs2qgRAAAATmhO0smlcAkshYNrqXEIOnjwoB544AH5+fnVRj0AAABwASVldv287YgkNkiF66lxCBoyZIjWrl1bG7UAAADARazcm6W8E2VqGOCtzrEhZpcD1EiN7wkaPny4/vKXv2jbtm1q3769bDZbtcdHjRrlsOIAAADgnGaf7Ao3OCFSHlaWwsG11DgE3XXXXZKkp59++rTHLBaLysvLL74qAAAAOK1yu6F52ypCEF3h4IpqHIJ+2xIbAAAA7mVtarYyC0oU7GtTj+ZhZpcD1BibpQIAAKBGKrvCDWwXKZsHbyfhei5o1C5evFgjR45Uy5Yt1bJlS40aNUpLlixxdG0AAABwMoZhaO7J+4HoCgdXVeMQ9Nlnn2ngwIHy8/PTAw88oAceeEC+vr664oorNHXq1NqoEQAAAE5i84FcHco9IT8vD13WqqHZ5QAXpMb3BD377LN68cUX9fDDD1cde+CBB/TKK6/omWee0bhx4xxaIAAAAJxHZVe4/m0j5GPzMLka4MLUeCZo7969Gjly5GnHR40apZSUFIcUBQAAAOdjGIbmbE2XJF2ZwFI4uK4ah6CYmBjNnz//tOM///yzYmJiHFIUAAAAnM/OIwVKzSqUl6dV/dtGmF0OcMFqvBzu0Ucf1QMPPKCNGzeqV69ekqRly5bpo48+0uuvv+7wAgEAAOAcZp+cBerbqqECvGv8NhJwGjUevffcc4+ioqL08ssv66uvvpIktWvXTl9++aWuuuoqhxcIAAAA5zCnqitctMmVABfngiL8mDFjNGbMGEfXAgAAACeVmnlcyYfz5Wm1aGA7lsLBtbG7FQAAAM6pcoPUni3C1MDPy+RqgItzXjNBoaGh2rlzpxo2bKiQkBBZLJaznpudne2w4gAAAOAcKltjD6ErHOqB8wpBr776qgIDA6v+//dCEAAAAOqXQzlF2rQ/RxaLNDgh0uxygIt2XiFo/PjxVf9/66231lYtAAAAcEJzTy6F6xIboohAH5OrAS5eje8J8vDwUEZGxmnHs7Ky5OHBrsEAAAD1DV3hUN/UOAQZhnHG48XFxfLy4iY5AACA+iSzoFhrUivu+R7CUjjUE+fdIvs///mPJMlisej9999XQEBA1WPl5eX65Zdf1LZtW8dXCAAAANPM23ZEdkPq0CRYTUL8zC4HcIjzDkGvvvqqpIqZoHfeeafa0jcvLy81a9ZM77zzjuMrBAAAgGnoCof66LxDUEpKiiSpf//+mj59ukJCQmqtKAAAAJgvt6hUy3dnSpKuTCQEof447xBUaeHChbVRBwAAAJzMguQjKrMbah0ZoBbhAef+AsBF1DgESdKBAwc0Y8YMpaWlqaSkpNpjr7zyikMKAwAAgLlmbznZFY6lcKhnahyC5s+fr1GjRql58+ZKTk5WYmKiUlNTZRiGOnXqVBs1AgAAoI4VlpRp8c6jkmiNjfqnxi2yH3/8cT322GPasmWLfHx8NG3aNO3fv1/9+vXTddddVxs1AgAAoI4t2nFUxWV2NQ31U7voQLPLARyqxiFo+/btuuWWWyRJnp6eKioqUkBAgJ5++mm98MILDi8QAAAAda9yg9ShiVGyWCwmVwM4Vo1DkL+/f9V9QNHR0dqzZ0/VY5mZmY6rDAAAAKYoLivXguQMSdIQusKhHqrxPUE9evTQ0qVL1a5dOw0bNkyPPvqotmzZounTp6tHjx61USMAAADq0LLdmSooLlNUkI8uadLA7HIAh6txCHrllVdUUFAgSZo0aZIKCgr05ZdfqlWrVnSGAwAAqAcqu8INSYiU1cpSONQ/NQ5BzZs3r/p/f39/vfPOOw4tCAAAAOYpK7dr3vYjkugKh/qrxvcEAQAAoP5alZKtnMJShfp7qWuzELPLAWrFec0EhYSEnHdXkOzs7IsqCAAAAOap7Ao3OD5Snh78vhz103mFoNdee62WywAAAIDZ7HZDc5NO3g9EVzjUY+cVgjZt2qRnnnlG/v7++uWXX9SrVy95etb4diIAAAA4sQ37jykjv1iB3p7q1SLM7HKAWnNec5xvvPFGVUe4/v37s+QNAACgHqpcCndFuwh5e3qYXA1Qe85rOqdZs2b6z3/+o8GDB8swDK1YsUIhIWe+Ua5v374OLRAAAAC1zzAMzT4Zgq5kKRzqufMKQf/+97/1pz/9Sc8995wsFovGjBlzxvMsFovKy8sdWiAAAABqX9KhPB04ViQfm1X9WkeYXQ5Qq84rBI0ePVqjR49WQUGBgoKCtGPHDkVE8JcDAACgvqhcCnd56wj5erEUDvVbjbobBAQEaOHChYqLi6MxAgAAQD0y52RXuKHtWQqH+q/GSaZfv36y2+3auXOnMjIyZLfbqz3OPUEAAACuZXdGvnZnFMjmYVH/tqz2Qf1X4xC0cuVKjRs3Tvv27ZNhGNUe454gAAAA11O5FK5Py4YK8rGZXA1Q+2q8DfCf/vQndenSRVu3blV2draOHTtW9XExrbOff/55WSwWPfTQQxf8HAAAAKg5usLB3dR4JmjXrl365ptv1LJlS4cVsWbNGr377rvq0KGDw54TAAAA57Y/u1BJh/JktUiD4glBcA81ngnq3r27du/e7bACCgoKdOONN+q99947695DAAAAqB2VS+G6x4Up1N/L5GqAulHjmaD7779fjz76qA4fPqz27dvLZqu+brSmszn33nuvhg8froEDB+qf//zn755bXFys4uLiqs/z8vIkSaWlpSotLa3RdR2t8vpm1wH3wZhDXWPMoS4x3urO7K3pkqTB8eFu/efNmHN9NfneWYzfdjc4B6v19Mkji8UiwzBq3Bjhiy++0LPPPqs1a9bIx8dHl19+uS655BK99tprZzx/4sSJmjRp0mnHp06dKj8/v/O+LgAAAKTcEumpdRW/E5/UqUwNvE0uCLgIhYWFGjdunHJzcxUUFPS759Z4JiglJeWCCzvV/v379eCDD2revHny8fE5r695/PHH9cgjj1R9npeXp5iYGA0ePPicL7S2lZaWat68eRo0aNBps2NAbWDMoa4x5lCXGG91Y8qqNEnJuiQmWOPGdDe7HFMx5lxf5Sqx81HjEBQbG1vTLzmjdevWKSMjQ506dao6Vl5erl9++UX//e9/VVxcLA+P6rsVe3t7y9v79F9R2Gw2pxmszlQL3ANjDnWNMYe6xHirXfOSj0qShrWP5s/5JMac66rJ9+28Q9CMGTPO67xRo0ad13lXXHGFtmzZUu3YbbfdprZt2+pvf/vbaQEIAAAAjnPseIlW7q3Y3uTKhGiTqwHq1nmHoNGjR5/znJrcExQYGKjExMRqx/z9/RUWFnbacQAAADjWvO1HVG43FB8dpKZh3FsN93LeIchut9dmHQAAAKhDc9ggFW6sxvcE1aZFixaZXQIAAEC9l3+iVEt3ZUqShhKC4IZqvFkqAAAAXNuC5AyVlNvVPNxfLSMCzC4HqHOEIAAAADczN6liKdzQxChZLBaTqwHqHiEIAADAjRSVlGvhydbYdIWDuyIEAQAAuJFfdh1VUWm5GjfwVWJjczebB8xyQSEoJydH77//vh5//HFlZ1f0l1+/fr0OHjzo0OIAAADgWKd2hWMpHNxVjbvDbd68WQMHDlRwcLBSU1N11113KTQ0VNOnT1daWpo++eST2qgTAAAAF6mkzK6ftx+RRGtsuLcazwQ98sgjuvXWW7Vr1y75+PhUHR82bJh++eUXhxYHAAAAx1mxN0v5J8oUHuitzk1DzC4HME2NQ9CaNWt09913n3a8cePGOnz4sEOKAgAAgOPN2ZouSRocHymrlaVwcF81DkHe3t7Ky8s77fjOnTsVHh7ukKIAAADgWOV2Qz8lVSyFG5pIVzi4txqHoFGjRunpp59WaWmpJMlisSgtLU1/+9vfdM011zi8QAAAAFy8NanZyjpeomBfm7o3DzW7HMBUNQ5BL7/8sgoKChQREaGioiL169dPLVu2VGBgoJ599tnaqBEAAAAXqbIr3KD4SNk82CUF7q3G3eGCg4M1b948LV26VJs3b1ZBQYE6deqkgQMH1kZ9AAAAuEh2u6G5SSdbYyfQFQ6ocQiq1KdPH/Xp08eRtQAAAKAWbD6Yq/TcE/L38lCfVg3NLgcwXY1D0H/+858zHrdYLPLx8VHLli3Vt29feXh4XHRxAAAAuHizT3aF6982Qj423qMBNQ5Br776qo4eParCwkKFhFT0lz927Jj8/PwUEBCgjIwMNW/eXAsXLlRMTIzDCwYAAMD5MwxDc0/eD0RXOKBCje+K+9e//qWuXbtq165dysrKUlZWlnbu3Knu3bvr9ddfV1pamqKiovTwww/XRr0AAACogeTD+UrNKpS3p1WXt2E7E0C6gJmgJ554QtOmTVOLFi2qjrVs2VIvvfSSrrnmGu3du1cvvvgi7bIBAACcQGVXuL6tw+XvfcG3gwP1So1ngtLT01VWVnba8bKyMh0+XPGXrFGjRsrPz7/46gAAAHBRKkMQXeGAX9U4BPXv31933323NmzYUHVsw4YNuueeezRgwABJ0pYtWxQXF+e4KgEAAFBje48WaMeRfHlaLRrYLtLscgCnUeMQNHnyZIWGhqpz587y9vaWt7e3unTpotDQUE2ePFmSFBAQoJdfftnhxQIAAOD8zU06Iknq2SJMwX42k6sBnEeNF4ZGRUVp3rx5Sk5O1s6dOyVJbdq0UZs2barO6d+/v+MqBAAAwAWZc7I19pWJLIUDTnXBd8e1bdtWbdu2dWQtAAAAcJCDOUXadCBXFos0OJ4QBJzqgkLQgQMHNGPGDKWlpamkpKTaY6+88opDCgMAAMCFq9wbqGtsqMIDvU2uBnAuNQ5B8+fP16hRo9S8eXMlJycrMTFRqampMgxDnTp1qo0aAQAAUENzkk52hWMpHHCaGjdGePzxx/XYY49py5Yt8vHx0bRp07R//37169dP1113XW3UCAAAgBo4ml+sNanZkqQhhCDgNDUOQdu3b9ctt9wiSfL09FRRUZECAgL09NNP64UXXnB4gQAAAKiZeduOyDCkjk2C1biBr9nlAE6nxiHI39+/6j6g6Oho7dmzp+qxzMxMx1UGAACACzL7ZFc4ZoGAM6vxPUE9evTQ0qVL1a5dOw0bNkyPPvqotmzZounTp6tHjx61USMAAADOU25hqVbsyZIkXZlACALOpMYh6JVXXlFBQYEkadKkSSooKNCXX36pVq1a0RkOAADAZD9vP6Iyu6E2kYFqHh5gdjmAU6pRCCovL9eBAwfUoUMHSRVL4955551aKQwAAAA1R1c44NxqdE+Qh4eHBg8erGPHjtVWPQAAALhAx4vL9MvOo5IIQcDvqXFjhMTERO3du7c2agEAAMBFWLTjqIrL7IoN81PbqECzywGcVo1D0D//+U899thjmjVrltLT05WXl1ftAwAAAOaYuemQpIpZIIvFYnI1gPOqcWOEYcOGSZJGjRpV7S+XYRiyWCwqLy93XHUAAAA4L/uzC/XTtor7ga6+tInJ1QDOrcYhaOHChbVRBwAAAC7CR8tTZTeky1o1VBuWwgG/q8YhqF+/frVRBwAAAC5Q/olSfblmvyTp9j5xJlcDOL8a3xMkSUuWLNFNN92kXr166eDBg5KkTz/9VEuXLnVocQAAADi3r9YeUEFxmVpGBKhfq3CzywGcXo1D0LRp0zRkyBD5+vpq/fr1Ki4uliTl5ubqX//6l8MLBAAAwNmV2w19uCxFknR77zhZrTREAM7lgrrDvfPOO3rvvfdks9mqjvfu3Vvr1693aHEAAAD4fT8lHdaBY0UK8bPp6k6NzS4HcAk1DkE7duxQ3759TzseHBysnJwcR9QEAACA8zR5acUs0I3dY+Vj8zC5GsA11DgERUVFaffu3acdX7p0qZo3b+6QogAAAHBum/bnaO2+Y7J5WHRLz1izywFcRo1D0F133aUHH3xQq1atksVi0aFDhzRlyhQ99thjuueee2qjRgAAAJxB5SzQyI6NFBHkY3I1gOuocYvs//u//5PdbtcVV1yhwsJC9e3bV97e3nrsscd0//3310aNAAAA+I1DOUX6YUu6JOkO2mIDNVLjEGSxWPSPf/xDf/nLX7R7924VFBQoPj5eAQEBtVEfAAAAzuDjFakqtxvq0TxUCY2CzS4HcCk1Xg732WefqbCwUF5eXoqPj1e3bt0IQAAAAHXoeHGZPl+VJkm6ow/3ZAM1VeMQ9PDDDysiIkLjxo3Tjz/+qPLy8tqoCwAAAGcxbf0B5Z0oU7MwP13RNsLscgCXU+MQlJ6eri+++EIWi0Vjx45VdHS07r33Xi1fvrw26gMAAMAp7HZDH5xsiHB7HzZHBS5EjUOQp6enRowYoSlTpigjI0OvvvqqUlNT1b9/f7Vo0aI2agQAAMBJ85MzlJpVqCAfT13TqYnZ5QAuqcaNEU7l5+enIUOG6NixY9q3b5+2b9/uqLoAAABwBpOX7pUk3dC9qfy9L+qtHOC2ajwTJEmFhYWaMmWKhg0bpsaNG+u1117TmDFjlJSU5Oj6AAAAcNLWg7lauTdbnlaLbu3VzOxyAJdV418f/OEPf9CsWbPk5+ensWPH6sknn1TPnj1rozYAAACcovJeoGHtoxUd7GtyNYDrqnEI8vDw0FdffaUhQ4bIw8Oj2mNbt25VYmKiw4oDAABAhYy8E5q5+ZAkNkcFLlaNQ9CUKVOqfZ6fn6/PP/9c77//vtatW0fLbAAAgFrwyYp9Ki031CU2RB1jGphdDuDSLuieIEn65ZdfNH78eEVHR+ull17SgAEDtHLlSkfWBgAAAElFJeWasmqfJOnOy5gFAi5WjWaCDh8+rI8++kiTJ09WXl6exo4dq+LiYn333XeKj4+vrRoBAADc2vQNB3SssFQxob4aFB9ldjmAyzvvmaCRI0eqTZs22rx5s1577TUdOnRIb7zxRm3WBgAA4PZO3Rz11l5x8mBzVOCinfdM0OzZs/XAAw/onnvuUatWrWqzJgAAAJy0eNdR7Tl6XAHenhrbhc1RAUc475mgpUuXKj8/X507d1b37t313//+V5mZmbVZGwAAgNurnAX6Q9cYBfrYTK4GqB/OOwT16NFD7733ntLT03X33Xfriy++UKNGjWS32zVv3jzl5+fXZp0AAABuJ/lwnpbsypTVIo1nc1TAYWrcHc7f31+33367li5dqi1btujRRx/V888/r4iICI0aNao2agQAAHBLlbNAVyZGKSbUz+RqgPrjgltkS1KbNm304osv6sCBA/r8888dVRMAAIDbyywo1ncb2RwVqA0XFYIqeXh4aPTo0ZoxY4Yjng4AAMDtfbZyn0rK7LokpoE6NQ0xuxygXnFICAIAAIDjnCgt16crKjZHvaNPnCwW2mIDjkQIAgAAcDIzNh5S1vESNQr20dBENkcFHI0QBAAA4EQMw9AHyyoaIozv1UyeHrxdAxyNv1UAAABOZNnuLCUfzpefl4f+0K2p2eUA9RIhCAAAwIm8v3SvJGlslxgF+7I5KlAbCEEAAABOYndGvhbtOCqLRbqtdzOzywHqLUIQAACAk/hgWaokaWC7SMWG+ZtbDFCPEYIAAACcQPbxEk1ff0ASm6MCtY0QBAAA4ASmrtqnE6V2JTYOUve4ULPLAeo1QhAAAIDJSsrs+oTNUYE6QwgCAAAw2azNh5SRX6yIQG8Nb9/I7HKAeo8QBAAAYCLDMDR56a+bo3p58vYMqG38LQMAADDRyr3ZSjqUJx+bVePYHBWoE4QgAAAAE1XOAl3TqYlC/L1MrgZwD4QgAAAAk6RmHtf85COSpNtpiw3UGUIQAACAST5cliLDkPq3CVeL8ACzywHcBiEIAADABLmFpfpqbcXmqHde1tzkagD3QggCAAAwwedr0lRUWq62UYHq1SLM7HIAt0IIAgAAqGOl5XZ9vDxVUsW9QGyOCtQtQhAAAEAdm731sNJzT6hhgJdGdWRzVKCuEYIAAADqkGEYmrxkryTp5h7N5GPzMLkiwP2YGoLefvttdejQQUFBQQoKClLPnj01e/ZsM0sCAACoVev2HdOmA7ny8rTqxh5sjgqYwdQQ1KRJEz3//PNat26d1q5dqwEDBuiqq65SUlKSmWUBAADUmsrNUcdc0lgNA7xNrgZwT55mXnzkyJHVPn/22Wf19ttva+XKlUpISDCpKgAAgNqxP7tQc5MOS2JzVMBMpoagU5WXl+vrr7/W8ePH1bNnzzOeU1xcrOLi4qrP8/LyJEmlpaUqLS2tkzrPpvL6ZtcB98GYQ11jzKEu1dfxNnnJHtkNqU/LMDUP86l3r8+V1dcx505q8r2zGIZh1GIt57Rlyxb17NlTJ06cUEBAgKZOnaphw4ad8dyJEydq0qRJpx2fOnWq/Pz8artUAACAC3aiTHpqvYeKyy36U9tytQsx9S0YUO8UFhZq3Lhxys3NVVBQ0O+ea3oIKikpUVpamnJzc/XNN9/o/fff1+LFixUfH3/auWeaCYqJiVFmZuY5X2htKy0t1bx58zRo0CDZbDZTa4F7YMyhrjHmUJfq43j7cPk+/Wv2DrUI99fs+3uxN5CTqY9jzt3k5eWpYcOG5xWCTF8O5+XlpZYtW0qSOnfurDVr1uj111/Xu+++e9q53t7e8vY+/QZCm83mNIPVmWqBe2DMoa4x5lCX6st4Kyu365OVaZKkO/o0l5eXl8kV4Wzqy5hzRzX5vjndPkF2u73abA8AAICr+2nbER04VqQQP5uu7tTY7HIAt2fqTNDjjz+uoUOHqmnTpsrPz9fUqVO1aNEizZ0718yyAAAAHKqyLfZNPWLZHBVwAqaGoIyMDN1yyy1KT09XcHCwOnTooLlz52rQoEFmlgUAAOAwG/fnaN2+Y7J5WHRzj1izywEgk0PQ5MmTzbw8AABAraucBRrZsZEignxMrgaA5IT3BAEAANQXh3KK9OOWdEnSHWyOCjgNQhAAAEAt+Xh5qsrthno2D1NCo2CzywFwEiEIAACgFhwvLtPU1ZVtsZkFApwJIQgAAKAWfLPugPJPlCmuob8GtI0wuxwApyAEAQAAOFi53dCHyyoaItzWu5msVovJFQE4FSEIAADAweZvP6LUrEIF+9p0becmZpcD4DcIQQAAAA5W2Rb7hm5N5edl6o4kAM6AEAQAAOBAWw/malVKtjytFo3vxeaogDMiBAEAADjQBydngYa1j1Z0sK/J1QA4E0IQAACAgxzJO6EZmw5Jku68jLbYgLMiBAEAADjIJytSVWY31LVZiDo0aWB2OQDOghAEAADgAEUl5Zqyis1RAVdACAIAAHCA6RsOKKewVDGhvhoUH2V2OQB+ByEIAADgItntRlVb7Nt6xcmDzVEBp0YIAgAAuEiLdx7V3qPHFejtqbFdY8wuB8A5EIIAAAAuUuUs0PVdYxTgzeaogLMjBAEAAFyE5MN5Wro7U1aLNL5XM7PLAXAeCEEAAAAXYfKSilmgoYnRign1M7kaAOeDEAQAAHCBjuYX6/uNFZuj3k5bbMBlEIIAAAAu0Gcr96mk3K5LYhqoc2yI2eUAOE+EIAAAgAtworRcn63cJ4nNUQFXQwgCAAC4AN9vPKis4yVq3MBXQxPZHBVwJYQgAACAGjKMXzdHHd8rVp4evKUCXAl/YwEAAGpo6e5M7TxSID8vD13ftanZ5QCoIUIQAABADVXOAo3tEqNgX5vJ1QCoKUIQAABADezOyNeiHUdlsUi39W5mdjkALgAhCAAAoAYmL02VJA1qF6nYMH9ziwFwQQhBAAAA5yn7eImmrz8gibbYgCsjBAEAAJynqav2qbjMrsTGQeoWF2p2OQAuECEIAADgPBSXlevjFb9ujmqxWEyuCMCFIgQBAACch1mb0nU0v1iRQd4a3r6R2eUAuAiEIAAAgHM4dXPUW3o2k5cnb6EAV8bfYAAAgHNYuTdb29Lz5GOz6sbubI4KuDpCEAAAwDlMXrpXknRNpyZq4OdlcjUALhYhCAAA4HekZB7X/OQMSdLttMUG6gVCEAAAwO/4cFmKDEMa0DZCLcIDzC4HgAMQggAAAM4it7BUX69lc1SgviEEAQAAnMXU1WkqKi1X26hA9WoRZnY5AByEEAQAAHAGpeV2fbw8VRKbowL1DSEIAADgDH7ckq7DeSfUMMBboy5hc1SgPiEEAQAA/Mapm6Pe3CNW3p4eJlcEwJEIQQAAAL+xdt8xbT6QKy9Pq27sweaoQH1DCAIAAPiNyUsqZoGuvrSxGgZ4m1wNAEcjBAEAAJxif3ahftp2WBKbowL1FSEIAADgFB8uS5XdkC5r1VCtIwPNLgdALSAEAQAAnJR3olRfrkmTxOaoQH1GCAIAADjpqzX7dbykXK0iAtSvdbjZ5QCoJYQgAAAASWXldn24LFVSxb1AbI4K1F+EIAAAAEk/bTuigzlFCvX30phLG5tdDoBaRAgCAACQ9P6SvZKkG7s3lY+NzVGB+owQBAAA3N6GtGNan5YjLw+rbu4Za3Y5AGoZIQgAALi9yUsrNkcd2bGRIgJ9TK4GQG0jBAEAALd2MKdIs7dWbI5KW2zAPRCCAACAW/t4earK7YZ6Ng9TfKMgs8sBUAcIQQAAwG0dLy7T56vZHBVwN4QgAADgtr5eu1/5J8oU19BfA9pGmF0OgDpCCAIAAG6p3G7ow+WpkqTbezeT1crmqIC7IAQBAAC39PP2I9qXVahgX5uu6dzE7HIA1CFCEAAAcEuVbbFv6NZUfl6eJlcDoC4RggAAgNvZejBXq1Oy5Wm1aHwvNkcF3A0hCAAAuJ3KWaDhHaIVHexrcjUA6hohCAAAuJXDuSc0c9MhSbTFBtwVIQgAALiVT1akqsxuqGuzEHVo0sDscgCYgBAEAADcRlFJuaZWbY7a3ORqAJiFEAQAANzGtPUHlFNYqqahfhoUH2l2OQBMQggCAABuwW439MHJhgi39momDzZHBdwWIQgAALiFRTsztDfzuAK9PTW2a4zZ5QAwETuDAQDgAGXldmUXlujY8VJlHS/WseOlyj5erNyiUnVtFqruzcPMLtHtVbbF/kO3GAV48xYIcGf8BABcUNKhXE1duU8H9lvV8ki+EpqEml0SUK8YhqHCknJlHy857SPreImOVf638NfjuUWlv/ucIzpE64nh8YoK9qmjV4FTbU/P07LdWbJapPG9mpldDgCTEYIAF2G3G5qfnKHJS/dq5d7sk0etWvzfFerWLFQ39miqoYnR8vJklSvwW+V2QzmFvwk0hSXKLjg9zFR+FJfZa3wdi0Vq4GtTqL9X1YfVYtHcpMOatTldC5Mz9NDA1rq1dzPZPPi7WpcqZ4GGJkarSYifydUAMBshCHByx4vLNG39AX2wNEWpWYWSJA+rRVcmROrAwUPamuOh1anZWp2arWcCtmlslxiN696Uf+RRrxWVlFeFmOzCEmUfL1b2yeVnZ5q9ySkqlWHU/DpenlaFnRJoQv29FOLnVXEswEuhftUfa+Dndcab7bcezNVT32/V+rQcPfvjdn21dr+evipRPVuwRK4uZOSf0IyNFZuj3s7mqABECAKc1qGcIn28IlWfr0pT3okySVKgj6fGdW+q8T2bKdzfUz/+eECd+lyuaRvS9fnqNB3JK9Zbi/bo7cV7NKBNhG7qGat+rcJlpQMSnJjdbii3qPRkmDm/j6LS8gu6VvApszS/F2YqP/y8PGSxXPzfn8TGwfrmT730zfoDen52snZlFOiG91Zq9CWN9Pdh7RQRxBK52vTZyjSVlNt1adMG6hwbYnY5AJwAIQhwMhv352jy0hT9uCVd5faKX103C/PTbb3jdG3nJvI/eTNvaWnF/QdRQT56aGBr3du/peZvP6LPVqZp6e5MzU/O0PzkDMWE+mpct1iN7dJEYQHepr0uuI/isjPfS3Omj2OFJTpWWFo11mvC5mH5NcwEeCnU31uhfraK//pX/DfE36Ywf++TszQ2U5egWa0Wje0So8HxkXrppx2asipN3208pJ+3Z+jhQa01vmesPFki53AnSss1ZeU+SdIdzAIBOIkQBDiBcruhn5IOa/LSFK3dd6zqeI/mobqjT3MNaBtxzv0sbB5WXZkYrSsTo7X3aIGmrErT12v3a392kV6Yk6xX5+3UsPZRurlnrDo1DXHIb7fhPkrL7dp0IFdbsi06vu6gck+UV1+CVljx32PHS1VQXHZB1wj09qyYlfE/fWYmxN/rtGVpAd6eLjmOG/h56Z+j22tslxg9+X2SNu3P0TOztunrtfv1zOhEdW1GoxNH+m7DQWUdL1HjBr66MiHK7HIAOAlCEGCi/BOl+nLNfn20PFUHjhVJqvjt9siOjXR77zglNg6+oOdtHh6gJ0fE67HBbTRz8yFNWblPmw7k6ruNh/TdxkNqGxWom3vGavQljatmloDfKrcbWpWSpVmb0zV7S7qOFZZK8pB2JJ3zaz2sll+Xm50jzFTO5rhbU48OTRro23t66cu1+/XCnGQlH87Xde+s0NWdGuvxoe0UHsjM7cUyDEMfLKtoiDC+FzNtAH7Fux/ABPuzC/XhslR9tXZ/1W/NQ/xsurF7rG7pGeuw+wN8vTw0tkuMxnaJ0eYDOfps5T59v/GQkg/n6x/fbtVzPybr6k6NdVOPWLWODHTINeHa7HZD69OOadbmdP2wJV1H84urHmvga1OQtURxjcMVFuCtsGqh5tclaKF+Xgrydc1ZmrpmtVp0Q7emujIhSi/O3aEv1qRp+vqDmrftiB4b3EY3dm/KG/eLsGRXpnYeKZC/l4eu79rU7HIAOBFCEFBHDMPQun3HNHlpiuYmHVblLRAtIwJ0e+84jbm0sXy9PGrt+h2aNNCL1zbQP4bF65v1BzRl5T7tzTyuT1bs0ycr9qlbXKhu6hGrKxOi3O438u7OMAxtOZirmZsO6YfN6TqUe6LqsSAfTw1NjNaIjtHqEhOkn+bO0bBhnWSz2UysuP4J8ffSc1e31/VdY/Tkd1u15WCuJsxI0pdrKpbIcTP/halsi31dlxgF+zJmAfyKEATUstJyu37ckq4PlqZo04HcquOXtWqoO/rEqW8dd28L9rPpjj5xur13My3fk6VPV+zTvO1HtDolW6tTstUwwEvXd43RDd1os12fGYah5MP5mrX5kGZuSldadmHVYwHenhoUH6mRHaPVp2V4VSiubMaB2nNJTAN9d29vfb46Tf+eu0Pb0vN0zdvLNbZLE/3tyrY0N6mBXUfytXjnUVks0m29m5ldDgAnQwgCakluYammrk7TJytSlX7yN+tenlaNuaSxbu8TpzZR5i4/s1gs6t2yoXq3bKjDuSf0+eo0fbGmos32mwv36O1FezSgbYRu7EGb7fpkd0aBZm0+pFmb07U7o6DquI/NqivaRWpkh0a6vE24fGy1NyuJ3+dhteimHrEamhilF+Yk66u1B/TV2gOas/Ww/nJlW43r1vScjVKgqnuBBrWLVGyYv8nVAHA2hCDAwVIyj+vDZSn6eu2Bqr1MGgZ46eYezXRjj6Zq6IS/yY0K9tHDg1rrvgEt9fO2I/ps1T4t252ln7dn6OftGWoa6qdx3ZtqbJcYhfp7mV0uaigtq1AzTwaf7el5Vce9PKy6vE24RnRspCvaRtAkw8mEBXjrxWs76vquTfXkd1u1LT1PT363VV+dXCJ3SUwDs0t0WtnHSzR9/UFJtMUGcGb8iwc4gGEYWrE3Sx8sTdH85IyqnenbRgXq9j5xGtWxkUv8Zt3mYdXQ9tEa2j5ae44WaMrKNH2zbr/Ssgv1/OxkvTJvp4a3j9ZNPWLVqWkDbnx3Yum5Rfphc7pmbjpUbRmmp9WiPq0aamSHRhqUEKkgH+6TcHadY0M0477emrIqTS/9tENbDuZqzFvL9IeuMfrrkLYK4RcTp5mycp+Ky+xq3zhY3eJoOQ7gdIQg4CKUlNk1c9MhTV6aom2n/IZ9QNsI3dEnTr1ahLlsUGgRHqCnRsbrL0PaaOamQ/ps1T5tPpCrbzcc1LcbDqpddJBu7hGrqy5pxAyCk8jIP6HZWw5r1uZDWpP6635TVovUs0WYRnRopCsTonjT7II8Pawa36uZhrWP1vOzkzVt/QF9vnq/Zm89rL9d2VbXd4lhyepJxWXl+njFr5ujuurPYAC1i3cuwAXIPl6iKSv36ZOV+6paCPvYrLqmUxPd1jtOLSMCTK7QcXy9PDS2a4zGdo3Rpv0VbbZnbDqk7el5+vu3W/SvH7frmpNttlvRZrvOHTteotlbK4LPyr1ZVV0HJalbs1CN6BitoYnR7DlTT4QHeuvlsR11fdcYPfX9ViUfztfj07foizX79c+rEtW+yYXtLVafzNyUrsyCYkUGeWtY+2izywHgpAhBQA3szsjX5KWpmr7+gIrL7JKkyCBv3dKzmcZ1a1rvf8PeMaaBOsY00D+Gt9M36w5oyqo0pWQe18cr9unjk222b+4RqyG02a5VeSdK9VPSEc3cdEjLdmeq7JTk0zGmgUZ2iNbwDtGKDvY1sUrUpm5xoZp1fx99vGKfXp23U5v252jUm0t1Y/ememxwGzXwq98/i87GMIyqtti39GzGzyEAZ2VqCHruuec0ffp0JScny9fXV7169dILL7ygNm3amFkWUI1hGFqyK1OTl6Zo8c6jVccTGwfpzj7NNax9tNv9Q9vAz0t3XtZct/eO0/I9Wfps5W/bbHvrD11jdEP3pmrcgDfijnC8uEw/bz+iWZvTtXjHUZWU26sei48O0oiO0RrRvpGahtHW3F14elh1R584jewQrX/9uF3fbTykz1am6ccth/V/V7bVtZ2buN0SuRV7s7Q9PU++Ng/d2J3NUQGcnakhaPHixbr33nvVtWtXlZWV6e9//7sGDx6sbdu2yd+fdpYw14nScn234aA+WJainUcqWglbLBXtVu/oE6ducaFuv9bcevIm+z6tfm2z/fnqNGXkF+u/C3frrUW7NaBtpG7q0bTO90OqD06UlmvRjgzN3JSu+clHdKL01+DTMiJAIzs00oiO0WoRXn+WX6LmIoJ89NofLtX1XZvqqe+3aldGgf46bbO+WJOmZ0YnKqGR+yyRm7ykYhboms6N3XY2DMD5MTUEzZkzp9rnH330kSIiIrRu3Tr17dv3tPOLi4tVXFxc9XleXsWN6KWlpaZv4ld5fbPrwMXLLCjWlFX7NXXNfmUfr/h++nl56NpOjXVLz6aKDa34TXtZWZmZZTrdmAvz89B9l8fp7stiNT/5qKau3q8Ve7P18/Yj+nn7EcWE+OqGbk10zaWNabP9O0rK7Fq6J0s/bjmsn7dn6HhJedVjTUN9Nbx9lIYnRql1ZEBVCK+rMeBsYw7VdWkapO//3EOfrEzTGwv2aH1ajka+UbFE7qEBLRTk61qdAGs63lIyj2t+coYk6eZuMYxT1Bg/41xfTb53FsMwjHOfVjd2796tVq1aacuWLUpMTDzt8YkTJ2rSpEmnHZ86dar8/FgCgotz8Li0KN2qdZkWlRsVby5DvAz1jbarR4QhP+6gq7EjRdKyw1atPmpRUXnFn6mnxdClYYZ6R9nVLKBids3dlRvSrlyL1mdatDn71z8rqWIMXhpmqFNDu5r48+eF85NTLH23z6oNWRVLdQNshq6KtatrQ6PejqGv91q19IhV8Q3surud/dxfAKDeKSws1Lhx45Sbm6ugoKDfPddpQpDdbteoUaOUk5OjpUuXnvGcM80ExcTEKDMz85wvtLaVlpZq3rx5GjRokGw21/ptmzuz2w0t3pWpj5bv0/K92VXHL4kJ1m09YzU4PkKeHs55v48rjbnCkjL9sOWwpq4+oK2Hfm0l3i4qUOO6xWhkhyi3a7Ndbje0dt8x/bDlsOZuO1I16yhJ4QFeGpoYpeHto3RJk2CnWUboSmMOFZbvydKkWcnam3lcktQltoEmjminNlHO38mxJuMtp7BUfV9arKJSuz65rbN6Ng+roypRn/AzzvXl5eWpYcOG5xWCnOZdx7333qutW7eeNQBJkre3t7y9T2/zarPZnGawOlMtOLvCkjJNW39QHy5L0d6jFW8OrBZpaGK0bu8Tp86xISZXeP5cYcwF22wa1yNO43rEadP+HH26cp9mbjqk7Yfz9eSMbXpx7k5d7QZttg3D0Pq0HM3afEg/bE5XRv6vv9QJ9a8IPiM6NFK3uFB5OEnwORNXGHOo0K9tlOa0jNDkpSn6z/xdWrsvR1e9vVLjezbTw4NaKdAFNss9n/H29YZ9Kiq1q21UoC5rHen292vi4vAzznXV5PvmFCHovvvu06xZs/TLL7+oSZMmZpeDeuxw7gl9siJVU1alKbeo4jfvgd6e+kO3GI3v1UxNQlhWWdsq22w/cZY2293jQnVzz1gNjq8fbbYNw9DWg3matfmQZm1O18GcoqrHgnw8NSQhSiM7NlKvFmFOO+sI1+bladU9l7fQVZc00j9/2KYftxzWB8tSNHPzIT0xvJ1GdWzk0qGhtNyuT5azOSqAmjE1BBmGofvvv1/ffvutFi1apLi4ODPLQT225UCuJi/dq1mb06v2VIkJ9dXtveN0XZcYBbjZUixncGqb7WV7MivabG87olUp2Vp1ss32Dd1idEO3pmrkgm22dxzO18xNhzRr8yGlZhVWHff38tCg+EiN6NBIl7VuKG9PDxOrhDtp1MBXb93YWb/sPKoJM5KUknlcD36xUVNXVXSRa+2is7A/bknX4bwTahjgrVGXNDK7HAAuwtR3fvfee6+mTp2q77//XoGBgTp8+LAkKTg4WL6+rvemB86l3G5o3rYj+mBpilan/nq/T7dmobq9T5wGxUc69ZIjd2G1WnRZq3Bd1ipc6blF+nz1fn1xss32Gwt2682FFW22b+4Zq8taNnSa+2POZO/RAs3anK6Zmw5pV0ZB1XEfm1VXtI3UiA7R6t82Qj42gg/M07d1uOY8dJneX5KiNxbs0qqUbA17fYlu691MDw5s7VK/FKq+OWosv1QAcN5M/Un39ttvS5Iuv/zyasc//PBD3XrrrXVfEOqFguIyfbVmvz5anqq07IrfwHtaLRrRIVp39Gmu9k3cZ88MVxMd7KtHBrXW/QNaat62I/ps5T4t35NV1WY7NsxPN3Zvqus6xyjESdps788u1KzN6Zq1+ZCSTmn64OVhVd/W4RrZMVoD20W6XeMHODdvTw/d27+lRnVspGdmbdNP247ovSUpmrHpkJ4YHq8RHaJdYlnZmtRj2nwgV16eVjZHBVAjpi+HAxzlwLFCfbw8VV+s3q/84oo9fIJ9bRrXvanG92ymqGAfkyvE+bJ5WDWsfbSGtY/W7owCTVm1T9+sO6B9WYX614/JeumnnRrRIVo39YjVpTEN6vzN2uHcE1X3+Gzcn1N13MNqUZ+WDTWiQ7QGJ0Qp2MX2ZYH7iQn10/9u6aKFyRmaODNJ+7IKdf/nG/TFmjRNGpWolhHOvRHv5KV7JUlXX9pYYQGnN04CgLPhV5Nweev2HdMHS1M0J+mwyk/e79O8ob9u6xOnazo1lp8Xw9yVtYwI0ISRCfrLkDaauemQPl25T1sP5mn6+oOavv6gEhoF6aYesbrqkka1+r3OLCjW7C3pmrkpXWv2ZavydzgWi9SzeZhGdGikKxOj2AgWLql/2wj1bBGmdxfv1VuLdmvZ7iwNff0X3dGnuR64oqVT/hxNyyrUT9uOSJJu78M9xQBqxvl+qgHnoazcrjlJhzV5aYo2pOVUHe/VIkx39IlT/zYRTn3vCGrOz8tT13dtqrFdYrTpQK4+XbGvagna49O36F8/bNc1nZvoph5N1TLCMTd45xSWaM7Ww5q1OV3L92TKfsrkdZfYEI3s2EhD20cpIpBZRrg+H5uHHhzYSmMubaxJM5M0PzlD7yzeoxkbD+rJEfG6MjHKqZbIfbg8RYZRcY+TqzZ1AGAeQhBcSm5Rqb5ck6aPl++rajXs5WHVqEsa6fbecYpvZO6muah9FotFl8Q00CUxDfTkiIo225+t3KfUrEJ9tDxVHy1PVY/mobqpx4W12c4/Uap5245o5qZDWrIrs6qboCR1bBKsER0aaXiHaJfsWAecj6Zhfpp8a1f9vO2IJs5M0oFjRbpnynpd1qqhJo1KUPNw85fI5Z0o1Vdr9kuqaIsNADVFCIJL2Jd1XB8uS9XXa/freEm5pIrNJW/qEaubejTlN/Fu6rdttj9dsU8/bz+ilXuztXJvtsIDvfWHrudus11YUqb52zM0c9MhLdp5VCVl9qrH2kYFamTHRhrRIVqxYf518bIApzAwPlJ9WjXUWwt3653Fe7VkV6aufG2J/ti3ue7t31K+XuZ1YvtydcW/Ba0iAtS3VUPT6gDgughBcFqGYWh1SrYmL03RvO1Hqu7BaBURoDv6xGn0pY1pNQxJ1dtsH8op0her0/T5mv06ekqb7SvaRermHrHqc7LN9onSci3acVSzNh/S/O0ZKiotr3q+FuH+J4NPI6e/MRyoTT42Dz0yuI2u7tREE2YkafHOo/rvwt36dsNBPTUyXoPjI+t8iVxZuV0fLU+VVHEvkDMt0QPgOghBcDolZXb9sOWQJi9N0daDv7Yc7tc6XHf0idNlrRryjx7OqlEDXz0yuI3uv6KVfkqqaLO9Ym+W5m07onnbKtpst28crEU7jqrgZBdBSWoa6qcRHaI1smMjtY0KZIwBp2jW0F8f3dZVP207oqdnbtPBnCLd/ek69W8TromjEup0lnRu0hEdzClSqL+XxlzauM6uC6B+IQTBaeQUlmjKqjR9siJVR/KKJUnenlZd3amxbu8dp1bc+IoasHlYNbxDtIZ3iNbujHx9tjJN09ZXtNnel1Wxf1SjYB8N7xCtER0aqUOTYIIP8DssFouGJETpslYN9ebC3frfL3u1cMdRLXv1F/2pXwv9+fIWdTI7//7Jttg3dW/KagAAF4wQBFOVlNmVknlcn6xI1bT1B3SitOJejPBAb93SI1Y39oil5TAuWsuIQE0claC/XlnRZnt/dpEubxOuTk1D6CII1JCfl6f+MqStru7URBNnJGnJrkz9Z/4ufbvhgCaOTNAV7SJr7drr045pQ1qOvDysuqlnbK1dB0D9RwiCwxWXlSuzoESZ+cXKLKj8KNHR/GIdLSg+5XiJcotKq31tfHSQ7ugTpxEdo+XtyW/44FiVbbYBXLwW4QH65PZumr31sJ6ZtU37s4t0x8drNbBdhCaMTFBMqJ/Drzl5aYokaWTHRjTEAXBRCEE4LydKy6uCS+YZwszRyrCTX6y8E2XnfsJT2DwsJ+/3aa4ezUNZkgQALsJisWhY+2j1ax2u/yzYpclLUvTz9gwt2ZWpe/u31B/7NnfYkrWDOUWas/WwJNpiA7h4hCA3dqK0/DezMyWnzNwU62j+r6Env7jmwaZhgPfJD6+K/wZWfB4eWHEs/OTjwb42liQBgAvz9/bU40Pb6brOTfTkd0lasTdLr8zbqenrD2jiqARd3ibioq/x8fJUldsN9WoRxp5wAC4aIaieKSwpU2b+KTMzBcUnPz+hzPySasvTCmoYbLw8rBWBpjLMBHirYaDXKWHHW+EnPw/2tTGjAwBupmVEoKbe1V0zN6frn7O2KTWrULd+uEZDEiL15Ih4NQm5sCVyBcVl+nxVmiRmgQA4BiHIBRwvLqs2O3P0DPfbVD5WWFJ+7ic8hZen9WSY8VZ4gFf12ZtqMzfeCvLxJNgAAH6XxWLRqI6NNKBthF7/eac+WJaquUlHtHjnUd0/oJXuvCyuxvd8fr12v/KLy9S8ob/6O2BWCQAIQSYwDEPHS8pPLjf79d6ao6eEmVNncU7dxPF8+NisZ5ydqQwzp4acQG+CDQDA8QK8PfWP4fG6tnOMnvx+q1anZOvfc3do2roDmnRVgi5rFX5ez1NuN/ThslRJ0m29m7F8GoBDEIIcxDAMFZVJKZnHlXPCflqYOZpf/X6bylbQ58vX5lG19Cz81PtrznC/jb+XB8EGAOAU2kQF6ss/9tD3Gw/pnz9s197M47p58moNbx+tJ0a0U3Sw7+9+/YLko0rLLlSwr03XdG5SR1UDqO8IQQ7ypykbtWCHp7Rm2Xl/jZ+XxymzM6csRQusCDrhp9xv4+/NtwoA4JosFotGX9pYA9pF6NV5O/Xx8lT9sCVdC3dk6IErWun23nHy8rSe8Ws/WJ4qSRrXvan8vPi3EIBj8NPEQYL9bJIkf2+Pqq5nDU82DggP8KnWQKCyoQA/zAEA7iTIx6YJIxN0XecYPfX9Vq3dd0zPz07WN+sO6OlRCerVsmG18/cXSGv35cjTatH4ns3MKRpAvcS7cAd5Ymgb9bKlafTIwbLZbGaXAwCA04pvFKSv7u6p6RsO6rkft2t3RoHGvb9KIzs20j+GtVNUcMVGqAvTK2aHhneIrjoGAI5w5rln1FiQr01ejtkPDgCAes9qtejazk204LHLNb5nrKwWaeamQ7ri5UV6f8leHThWpA1ZFfe30hYbgKMRggAAgGmCfW2adFWiZtzXR5c2baDjJeX65w/bNfy/y2U3LOoS20AdmjQwu0wA9QwhCAAAmC6xcbCm/amXXrymg0L9var2vbutV6zJlQGoj7gnCAAAOAWr1aKxXWM0OCFS/12wS3v37NUVbdkcFYDjMRMEAACcSgM/L/1tSGtd1cwuDzZHBVALCEEAAAAA3AohCAAAAIBbIQQBAAAAcCuEIAAAAABuhRAEAAAAwK0QggAAAAC4FUIQAAAAALdCCAIAAADgVghBAAAAANwKIQgAAACAWyEEAQAAAHArhCAAAAAAboUQBAAAAMCtEIIAAAAAuBVCEAAAAAC3QggCAAAA4FYIQQAAAADcCiEIAAAAgFvxNLuAi2EYhiQpLy/P5Eqk0tJSFRYWKi8vTzabzexy4AYYc6hrjDnUJcYb6hpjzvVVZoLKjPB7XDoE5efnS5JiYmJMrgQAAACAM8jPz1dwcPDvnmMxzicqOSm73a5Dhw4pMDBQFovF1Fry8vIUExOj/fv3KygoyNRa4B4Yc6hrjDnUJcYb6hpjzvUZhqH8/Hw1atRIVuvv3/Xj0jNBVqtVTZo0MbuMaoKCgviLgzrFmENdY8yhLjHeUNcYc67tXDNAlWiMAAAAAMCtEIIAAAAAuBVCkIN4e3trwoQJ8vb2NrsUuAnGHOoaYw51ifGGusaYcy8u3RgBAAAAAGqKmSAAAAAAboUQBAAAAMCtEIIAAAAAuBVCEAAAAAC3Qgg6xXPPPaeuXbsqMDBQERERGj16tHbs2FHtnBMnTujee+9VWFiYAgICdM011+jIkSPVznnggQfUuXNneXt765JLLvnda+7evVuBgYFq0KCBg18NnF1djjfDMPTSSy+pdevW8vb2VuPGjfXss8/W1kuDk6rLMTd37lz16NFDgYGBCg8P1zXXXKPU1NRaemVwVo4Yc5s2bdINN9ygmJgY+fr6ql27dnr99ddPu9aiRYvUqVMneXt7q2XLlvroo49q++XBydTVeJs+fboGDRqk8PBwBQUFqWfPnpo7d26dvEY4DiHoFIsXL9a9996rlStXat68eSotLdXgwYN1/PjxqnMefvhhzZw5U19//bUWL16sQ4cO6eqrrz7tuW6//XZdf/31v3u90tJS3XDDDbrssssc/lrg/OpyvD344IN6//339dJLLyk5OVkzZsxQt27dauV1wXnV1ZhLSUnRVVddpQEDBmjjxo2aO3euMjMzz/g8qN8cMebWrVuniIgIffbZZ0pKStI//vEPPf744/rvf/9bdU5KSoqGDx+u/v37a+PGjXrooYd055138sbUzdTVePvll180aNAg/fjjj1q3bp369++vkSNHasOGDXX6enGRDJxVRkaGIclYvHixYRiGkZOTY9hsNuPrr7+uOmf79u2GJGPFihWnff2ECROMjh07nvX5//rXvxo33XST8eGHHxrBwcGOLh8uprbG27Zt2wxPT08jOTm51mqHa6qtMff1118bnp6eRnl5edWxGTNmGBaLxSgpKXH8C4HLuNgxV+nPf/6z0b9//6rP//rXvxoJCQnVzrn++uuNIUOGOPgVwJXU1ng7k/j4eGPSpEmOKRx1gpmg35GbmytJCg0NlVTx24HS0lINHDiw6py2bduqadOmWrFiRY2ee8GCBfr666/15ptvOq5guLTaGm8zZ85U8+bNNWvWLMXFxalZs2a68847lZ2d7dgXAJdTW2Ouc+fOslqt+vDDD1VeXq7c3Fx9+umnGjhwoGw2m2NfBFyKo8Zcbm5u1XNI0ooVK6o9hyQNGTKkxv82o36prfH2W3a7Xfn5+b97DpwPIegs7Ha7HnroIfXu3VuJiYmSpMOHD8vLy+u0+3ciIyN1+PDh837urKws3Xrrrfroo48UFBTkyLLhompzvO3du1f79u3T119/rU8++UQfffSR1q1bp2uvvdaRLwEupjbHXFxcnH766Sf9/e9/l7e3txo0aKADBw7oq6++cuRLgItx1Jhbvny5vvzyS/3xj3+sOnb48GFFRkae9hx5eXkqKipy7AuBS6jN8fZbL730kgoKCjR27FiH1Y/a52l2Ac7q3nvv1datW7V06VKHP/ddd92lcePGqW/fvg5/brim2hxvdrtdxcXF+uSTT9S6dWtJ0uTJk9W5c2ft2LFDbdq0cfg14fxqc8wdPnxYd911l8aPH68bbrhB+fn5euqpp3Tttddq3rx5slgsDr8mnJ8jxtzWrVt11VVXacKECRo8eLADq0N9U1fjberUqZo0aZK+//57RUREXPC1UPeYCTqD++67T7NmzdLChQvVpEmTquNRUVEqKSlRTk5OtfOPHDmiqKio837+BQsW6KWXXpKnp6c8PT11xx13KDc3V56envrggw8c9TLgImp7vEVHR8vT07MqAElSu3btJElpaWkXVzxcUm2PuTfffFPBwcF68cUXdemll6pv37767LPPNH/+fK1atcpRLwMuxBFjbtu2bbriiiv0xz/+UU888US1x6Kiok7rYnjkyBEFBQXJ19fXsS8GTq+2x1ulL774Qnfeeae++uqr05ZjwvkRgk5hGIbuu+8+ffvtt1qwYIHi4uKqPd65c2fZbDbNnz+/6tiOHTuUlpamnj17nvd1VqxYoY0bN1Z9PP300woMDNTGjRs1ZswYh70eOLe6Gm+9e/dWWVmZ9uzZU3Vs586dkqTY2NiLfBVwJXU15goLC2W1Vv/nxcPDQ1LFzCTch6PGXFJSkvr376/x48efsb1/z549qz2HJM2bN69G4xaur67GmyR9/vnnuu222/T5559r+PDhtfOCULtMbcvgZO655x4jODjYWLRokZGenl71UVhYWHXOn/70J6Np06bGggULjLVr1xo9e/Y0evbsWe15du3aZWzYsMG4++67jdatWxsbNmwwNmzYYBQXF5/xunSHc091Nd7Ky8uNTp06GX379jXWr19vrF271ujevbsxaNCgOn29MF9djbn58+cbFovFmDRpkrFz505j3bp1xpAhQ4zY2Nhq10L954gxt2XLFiM8PNy46aabqj1HRkZG1Tl79+41/Pz8jL/85S/G9u3bjTfffNPw8PAw5syZU6evF+aqq/E2ZcoUw9PT03jzzTernZOTk1OnrxcXhxB0Ckln/Pjwww+rzikqKjL+/Oc/GyEhIYafn58xZswYIz09vdrz9OvX74zPk5KScsbrEoLcU12Ot4MHDxpXX321ERAQYERGRhq33nqrkZWVVUevFM6iLsfc559/blx66aWGv7+/ER4ebowaNcrYvn17Hb1SOAtHjLkJEyac8TliY2OrXWvhwoXGJZdcYnh5eRnNmzevdg24h7oab2f7GTh+/Pi6e7G4aBbDMAzHzCkBAAAAgPPjniAAAAAAboUQBAAAAMCtEIIAAAAAuBVCEAAAAAC3QggCAAAA4FYIQQAAAADcCiEIAAAAgFshBAEAAABwK4QgAAAAAG6FEAQAcBqGYWjgwIEaMmTIaY+99dZbatCggQ4cOGBCZQCA+oQQBABwGhaLRR9++KFWrVqld999t+p4SkqK/vrXv+qNN95QkyZNHHrN0tJShz4fAMD5EYIAAE4lJiZGr7/+uh577DGlpKTIMAzdcccdGjx4sC699FINHTpUAQEBioyM1M0336zMzMyqr50zZ4769OmjBg0aKCwsTCNGjNCePXuqHk9NTZXFYtGXX36pfv36ycfHR1OmTDHjZQIATGQxDMMwuwgAAH5r9OjRys3N1dVXX61nnnlGSUlJSkhI0J133qlbbrlFRUVF+tvf/qaysjItWLBAkjRt2jRZLBZ16NBBBQUFeuqpp5SamqqNGzfKarUqNTVVcXFxatasmV5++WVdeuml8vHxUXR0tMmvFgBQlwhBAACnlJGRoYSEBGVnZ2vatGnaunWrlixZorlz51adc+DAAcXExGjHjh1q3br1ac+RmZmp8PBwbdmyRYmJiVUh6LXXXtODDz5Yly8HAOBEWA4HAHBKERERuvvuu9WuXTuNHj1amzZt0sKFCxUQEFD10bZtW0mqWvK2a9cu3XDDDWrevLmCgoLUrFkzSVJaWlq15+7SpUudvhYAgHPxNLsAAADOxtPTU56eFf9UFRQUaOTIkXrhhRdOO69yOdvIkSMVGxur9957T40aNZLdbldiYqJKSkqqne/v71/7xQMAnBYhCADgEjp16qRp06apWbNmVcHoVFlZWdqxY4fee+89XXbZZZKkpUuX1nWZAAAXwHI4AIBLuPfee5Wdna0bbrhBa9as0Z49ezR37lzddtttKi8vV0hIiMLCwvS///1Pu3fv1oIFC/TII4+YXTYAwAkRggAALqFRo0ZatmyZysvLNXjwYLVv314PPfSQGjRoIKvVKqvVqi+++ELr1q1TYmKiHn74Yf373/82u2wAgBOiOxwAAAAAt8JMEAAAAAC3QggCAAAA4FYIQQAAAADcCiEIAAAAgFshBAEAAABwK4QgAAAAAG6FEAQAAADArRCCAAAAALgVQhAAAAAAt0IIAgAAAOBWCEEAAAAA3Mr/A0VAtdI/K4eiAAAAAElFTkSuQmCC", "text/plain": [ "
" ] @@ -1841,29 +2334,877 @@ } ], "source": [ - "import pandas as pd\n", + "# NBVAL_SKIP\n", "import matplotlib.pyplot as plt\n", + "import pandas as pd\n", "\n", - "# Read the CSV file\n", - "df = pd.read_csv('/tmp/tmpco0s0o4_/LOdZoVp1inflation.csv')\n", + "# Load data\n", + "df = pd.read_csv(\"/tmp/tmpvzjigv7g/n2OzlTWhinflation.csv\")\n", "\n", - "# Extract the year and inflation rate from the CSV file\n", - "df['Year'] = pd.to_datetime(df['Year'], format='%Y')\n", - "df = df.rename(columns={'Jan': 'Jan Rate', 'Feb': 'Feb Rate', 'Mar': 'Mar Rate', 'Apr': 'Apr Rate', 'May': 'May Rate', 'Jun': 'Jun Rate', 'Jul': 'Jul Rate', 'Aug': 'Aug Rate', 'Sep': 'Sep Rate', 'Oct': 'Oct Rate', 'Nov': 'Nov Rate', 'Dec': 'Dec Rate'})\n", + "# Calculate average yearly inflation\n", + "df['Average'] = df[['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']].mean(axis=1)\n", "\n", - "# Calculate the average yearly inflation rate\n", - "df['Yearly Inflation'] = df[['Jan Rate', 'Feb Rate', 'Mar Rate', 'Apr Rate', 'May Rate', 'Jun Rate', 'Jul Rate', 'Aug Rate', 'Sep Rate', 'Oct Rate', 'Nov Rate', 'Dec Rate']].mean(axis=1)\n", - "\n", - "# Plot the average yearly inflation rate as a time series\n", - "plt.figure(figsize=(10, 6))\n", - "plt.plot(df['Year'], df['Yearly Inflation'], marker='o')\n", - "plt.title('Average Yearly Inflation Rate')\n", + "# Plot average yearly inflation as a time series\n", + "plt.figure(figsize=(10,6))\n", + "plt.plot(df['Year'], df['Average'])\n", + "plt.title('Average Yearly Inflation')\n", "plt.xlabel('Year')\n", - "plt.ylabel('Inflation Rate (%)')\n", + "plt.ylabel('Average Inflation')\n", "plt.grid(True)\n", "plt.show()" ] }, + { + "cell_type": "markdown", + "id": "jSfjNN9fMxtm", + "metadata": { + "id": "jSfjNN9fMxtm" + }, + "source": [ + "### 2.5. Using Model Context Protocol\n", + "\n", + "In this example, we will show how tools hosted in an MCP server can be configured to be used by the model.\n", + "\n", + "In the following steps, we will use the [filesystem tool](https://github.com/modelcontextprotocol/servers/tree/main/src/filesystem) to explore the files and folders available in the /content directory\n", + "\n", + "Use xterm module to start a shell to run the MCP server using the `supergateway` tool which can start an MCP tool and serve it over HTTP." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "67fDKVVpNuFb", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "67fDKVVpNuFb", + "outputId": "aec2e3cf-e1c3-4d09-d9dc-c4a2f1327e99" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting colab-xterm\n", + " Downloading colab_xterm-0.2.0-py3-none-any.whl.metadata (1.2 kB)\n", + "Requirement already satisfied: ptyprocess~=0.7.0 in /usr/local/lib/python3.11/dist-packages (from colab-xterm) (0.7.0)\n", + "Requirement already satisfied: tornado>5.1 in /usr/local/lib/python3.11/dist-packages (from colab-xterm) (6.3.3)\n", + "Downloading colab_xterm-0.2.0-py3-none-any.whl (115 kB)\n", + "\u001b[?25l \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/115.6 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.6/115.6 kB\u001b[0m \u001b[31m4.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hInstalling collected packages: colab-xterm\n", + "Successfully installed colab-xterm-0.2.0\n" + ] + } + ], + "source": [ + "!pip install colab-xterm #https://pypi.org/project/colab-xterm/\n", + "%load_ext colabxterm" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "giIA2M-ANUIM", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 839, + "resources": { + "https://localhost:10000/": { + "data": "PCFkb2N0eXBlIGh0bWw+PGh0bWw+PGhlYWQ+PG1ldGEgY2hhcnNldD0idXRmLTgiLz48c2NyaXB0IGRlZmVyPSJkZWZlciIgc3JjPSJtYWluLmpzIj48L3NjcmlwdD48L2hlYWQ+PGJvZHk+PGRpdiBpZD0idGVybWluYWwiPjwvZGl2PjwvYm9keT48L2h0bWw+", + "headers": [ + [ + "content-length", + "147" + ], + [ + "content-type", + "text/html; charset=UTF-8" + ] + ], + "ok": true, + "status": 200, + "status_text": "" + }, + "https://localhost:10000/in/Aw==": { + "data": "", + "headers": [ + [ + "content-length", + "0" + ], + [ + "content-type", + "text/html; charset=UTF-8" + ] + ], + "ok": true, + "status": 200, + "status_text": "" + }, + "https://localhost:10000/in/DA==": { + "data": "", + "headers": [ + [ + "content-length", + "0" + ], + [ + "content-type", + "text/html; charset=UTF-8" + ] + ], + "ok": true, + "status": 200, + "status_text": "" + }, + "https://localhost:10000/in/DQ==": { + "data": "", + "headers": [ + [ + "content-length", + "0" + ], + [ + "content-type", + "text/html; charset=UTF-8" + ] + ], + "ok": true, + "status": 200, + "status_text": "" + }, + "https://localhost:10000/in/G1syMDB+bnB4IC15IHN1cGVyZ2F0ZXdheSAtLXBvcnQgODAwMCAtLXN0ZGlvICducHggLXkgQG1vZGVsY29udGV4dHByb3RvY29sL3NlcnZlci1maWxlc3lzdGVtIC9jb250ZW50JxtbMjAxfg==": { + "data": "", + "headers": [ + [ + "content-length", + "0" + ], + [ + "content-type", + "text/html; charset=UTF-8" + ] + ], + "ok": true, + "status": 200, + "status_text": "" + }, + "https://localhost:10000/in/G1tB": { + "data": "", + "headers": [ + [ + "content-length", + "0" + ], + [ + "content-type", + "text/html; charset=UTF-8" + ] + ], + "ok": true, + "status": 200, + "status_text": "" + }, + "https://localhost:10000/in/IA==": { + "data": "", + "headers": [ + [ + "content-length", + "0" + ], + [ + "content-type", + "text/html; charset=UTF-8" + ] + ], + "ok": true, + "status": 200, + "status_text": "" + }, + "https://localhost:10000/in/Y2g=": { + "data": "", + "headers": [ + [ + "content-length", + "0" + ], + [ + "content-type", + "text/html; charset=UTF-8" + ] + ], + "ok": true, + "status": 200, + "status_text": "" + }, + "https://localhost:10000/in/YXI=": { + "data": "", + "headers": [ + [ + "content-length", + "0" + ], + [ + "content-type", + "text/html; charset=UTF-8" + ] + ], + "ok": true, + "status": 200, + "status_text": "" + }, + "https://localhost:10000/in/Yg==": { + "data": "", + "headers": [ + [ + "content-length", + "0" + ], + [ + "content-type", + "text/html; charset=UTF-8" + ] + ], + "ok": true, + "status": 200, + "status_text": "" + }, + "https://localhost:10000/in/Yw==": { + "data": "", + "headers": [ + [ + "content-length", + "0" + ], + [ + "content-type", + "text/html; charset=UTF-8" + ] + ], + "ok": true, + "status": 200, + "status_text": "" + }, + "https://localhost:10000/in/Zg==": { + "data": "", + "headers": [ + [ + "content-length", + "0" + ], + [ + "content-type", + "text/html; charset=UTF-8" + ] + ], + "ok": true, + "status": 200, + "status_text": "" + }, + "https://localhost:10000/in/aCA=": { + "data": "", + "headers": [ + [ + "content-length", + "0" + ], + [ + "content-type", + "text/html; charset=UTF-8" + ] + ], + "ok": true, + "status": 200, + "status_text": "" + }, + "https://localhost:10000/in/b3U=": { + "data": "", + "headers": [ + [ + "content-length", + "0" + ], + [ + "content-type", + "text/html; charset=UTF-8" + ] + ], + "ok": true, + "status": 200, + "status_text": "" + }, + "https://localhost:10000/in/bw0=": { + "data": "", + "headers": [ + [ + "content-length", + "0" + ], + [ + "content-type", + "text/html; charset=UTF-8" + ] + ], + "ok": true, + "status": 200, + "status_text": "" + }, + "https://localhost:10000/in/bw==": { + "data": "", + "headers": [ + [ + "content-length", + "0" + ], + [ + "content-type", + "text/html; charset=UTF-8" + ] + ], + "ok": true, + "status": 200, + "status_text": "" + }, + "https://localhost:10000/in/dA==": { + "data": "", + "headers": [ + [ + "content-length", + "0" + ], + [ + "content-type", + "text/html; charset=UTF-8" + ] + ], + "ok": true, + "status": 200, + "status_text": "" + }, + "https://localhost:10000/in/dQ==": { + "data": "", + "headers": [ + [ + "content-length", + "0" + ], + [ + "content-type", + "text/html; charset=UTF-8" + ] + ], + "ok": true, + "status": 200, + "status_text": "" + }, + "https://localhost:10000/main.js": { + "data": "LyohIEZvciBsaWNlbnNlIGluZm9ybWF0aW9uIHBsZWFzZSBzZWUgbWFpbi5qcy5MSUNFTlNFLnR4dCAqLwooKCk9Pnt2YXIgZT17MTAyOihlLHQscik9PnsidXNlIHN0cmljdCI7ci5kKHQse1o6KCk9PmF9KTt2YXIgaT1yKDgxKSxuPXIubihpKSxvPXIoNjQ1KSxzPXIubihvKSgpKG4oKSk7cy5wdXNoKFtlLmlkLCcvKipcbiAqIENvcHlyaWdodCAoYykgMjAxNCBUaGUgeHRlcm0uanMgYXV0aG9ycy4gQWxsIHJpZ2h0cyByZXNlcnZlZC5cbiAqIENvcHlyaWdodCAoYykgMjAxMi0yMDEzLCBDaHJpc3RvcGhlciBKZWZmcmV5IChNSVQgTGljZW5zZSlcbiAqIGh0dHBzOi8vZ2l0aHViLmNvbS9jaGpqL3Rlcm0uanNcbiAqIEBsaWNlbnNlIE1JVFxuICpcbiAqIFBlcm1pc3Npb24gaXMgaGVyZWJ5IGdyYW50ZWQsIGZyZWUgb2YgY2hhcmdlLCB0byBhbnkgcGVyc29uIG9idGFpbmluZyBhIGNvcHlcbiAqIG9mIHRoaXMgc29mdHdhcmUgYW5kIGFzc29jaWF0ZWQgZG9jdW1lbnRhdGlvbiBmaWxlcyAodGhlICJTb2Z0d2FyZSIpLCB0byBkZWFsXG4gKiBpbiB0aGUgU29mdHdhcmUgd2l0aG91dCByZXN0cmljdGlvbiwgaW5jbHVkaW5nIHdpdGhvdXQgbGltaXRhdGlvbiB0aGUgcmlnaHRzXG4gKiB0byB1c2UsIGNvcHksIG1vZGlmeSwgbWVyZ2UsIHB1Ymxpc2gsIGRpc3RyaWJ1dGUsIHN1YmxpY2Vuc2UsIGFuZC9vciBzZWxsXG4gKiBjb3BpZXMgb2YgdGhlIFNvZnR3YXJlLCBhbmQgdG8gcGVybWl0IHBlcnNvbnMgdG8gd2hvbSB0aGUgU29mdHdhcmUgaXNcbiAqIGZ1cm5pc2hlZCB0byBkbyBzbywgc3ViamVjdCB0byB0aGUgZm9sbG93aW5nIGNvbmRpdGlvbnM6XG4gKlxuICogVGhlIGFib3ZlIGNvcHlyaWdodCBub3RpY2UgYW5kIHRoaXMgcGVybWlzc2lvbiBub3RpY2Ugc2hhbGwgYmUgaW5jbHVkZWQgaW5cbiAqIGFsbCBjb3BpZXMgb3Igc3Vic3RhbnRpYWwgcG9ydGlvbnMgb2YgdGhlIFNvZnR3YXJlLlxuICpcbiAqIFRIRSBTT0ZUV0FSRSBJUyBQUk9WSURFRCAiQVMgSVMiLCBXSVRIT1VUIFdBUlJBTlRZIE9GIEFOWSBLSU5ELCBFWFBSRVNTIE9SXG4gKiBJTVBMSUVELCBJTkNMVURJTkcgQlVUIE5PVCBMSU1JVEVEIFRPIFRIRSBXQVJSQU5USUVTIE9GIE1FUkNIQU5UQUJJTElUWSxcbiAqIEZJVE5FU1MgRk9SIEEgUEFSVElDVUxBUiBQVVJQT1NFIEFORCBOT05JTkZSSU5HRU1FTlQuIElOIE5PIEVWRU5UIFNIQUxMIFRIRVxuICogQVVUSE9SUyBPUiBDT1BZUklHSFQgSE9MREVSUyBCRSBMSUFCTEUgRk9SIEFOWSBDTEFJTSwgREFNQUdFUyBPUiBPVEhFUlxuICogTElBQklMSVRZLCBXSEVUSEVSIElOIEFOIEFDVElPTiBPRiBDT05UUkFDVCwgVE9SVCBPUiBPVEhFUldJU0UsIEFSSVNJTkcgRlJPTSxcbiAqIE9VVCBPRiBPUiBJTiBDT05ORUNUSU9OIFdJVEggVEhFIFNPRlRXQVJFIE9SIFRIRSBVU0UgT1IgT1RIRVIgREVBTElOR1MgSU5cbiAqIFRIRSBTT0ZUV0FSRS5cbiAqXG4gKiBPcmlnaW5hbGx5IGZvcmtlZCBmcm9tICh3aXRoIHRoZSBhdXRob3JcJ3MgcGVybWlzc2lvbik6XG4gKiAgIEZhYnJpY2UgQmVsbGFyZFwncyBqYXZhc2NyaXB0IHZ0MTAwIGZvciBqc2xpbnV4OlxuICogICBodHRwOi8vYmVsbGFyZC5vcmcvanNsaW51eC9cbiAqICAgQ29weXJpZ2h0IChjKSAyMDExIEZhYnJpY2UgQmVsbGFyZFxuICogICBUaGUgb3JpZ2luYWwgZGVzaWduIHJlbWFpbnMuIFRoZSB0ZXJtaW5hbCBpdHNlbGZcbiAqICAgaGFzIGJlZW4gZXh0ZW5kZWQgdG8gaW5jbHVkZSB4dGVybSBDU0kgY29kZXMsIGFtb25nXG4gKiAgIG90aGVyIGZlYXR1cmVzLlxuICovXG5cbi8qKlxuICogIERlZmF1bHQgc3R5bGVzIGZvciB4dGVybS5qc1xuICovXG5cbi54dGVybSB7XG4gICAgcG9zaXRpb246IHJlbGF0aXZlO1xuICAgIC1tb3otdXNlci1zZWxlY3Q6IG5vbmU7XG4gICAgICAgICB1c2VyLXNlbGVjdDogbm9uZTtcbiAgICAtbXMtdXNlci1zZWxlY3Q6IG5vbmU7XG4gICAgLXdlYmtpdC11c2VyLXNlbGVjdDogbm9uZTtcbn1cblxuLnh0ZXJtLmZvY3VzLFxuLnh0ZXJtOmZvY3VzIHtcbiAgICBvdXRsaW5lOiBub25lO1xufVxuXG4ueHRlcm0gLnh0ZXJtLWhlbHBlcnMge1xuICAgIHBvc2l0aW9uOiBhYnNvbHV0ZTtcbiAgICB0b3A6IDA7XG4gICAgLyoqXG4gICAgICogVGhlIHotaW5kZXggb2YgdGhlIGhlbHBlcnMgbXVzdCBiZSBoaWdoZXIgdGhhbiB0aGUgY2FudmFzZXMgaW4gb3JkZXIgZm9yXG4gICAgICogSU1FcyB0byBhcHBlYXIgb24gdG9wLlxuICAgICAqL1xuICAgIHotaW5kZXg6IDU7XG59XG5cbi54dGVybSAueHRlcm0taGVscGVyLXRleHRhcmVhIHtcbiAgICBwYWRkaW5nOiAwO1xuICAgIGJvcmRlcjogMDtcbiAgICBtYXJnaW46IDA7XG4gICAgLyogTW92ZSB0ZXh0YXJlYSBvdXQgb2YgdGhlIHNjcmVlbiB0byB0aGUgZmFyIGxlZnQsIHNvIHRoYXQgdGhlIGN1cnNvciBpcyBub3QgdmlzaWJsZSAqL1xuICAgIHBvc2l0aW9uOiBhYnNvbHV0ZTtcbiAgICBvcGFjaXR5OiAwO1xuICAgIGxlZnQ6IC05OTk5ZW07XG4gICAgdG9wOiAwO1xuICAgIHdpZHRoOiAwO1xuICAgIGhlaWdodDogMDtcbiAgICB6LWluZGV4OiAtNTtcbiAgICAvKiogUHJldmVudCB3cmFwcGluZyBzbyB0aGUgSU1FIGFwcGVhcnMgYWdhaW5zdCB0aGUgdGV4dGFyZWEgYXQgdGhlIGNvcnJlY3QgcG9zaXRpb24gKi9cbiAgICB3aGl0ZS1zcGFjZTogbm93cmFwO1xuICAgIG92ZXJmbG93OiBoaWRkZW47XG4gICAgcmVzaXplOiBub25lO1xufVxuXG4ueHRlcm0gLmNvbXBvc2l0aW9uLXZpZXcge1xuICAgIC8qIFRPRE86IENvbXBvc2l0aW9uIHBvc2l0aW9uIGdvdCBtZXNzZWQgdXAgc29tZXdoZXJlICovXG4gICAgYmFja2dyb3VuZDogIzAwMDtcbiAgICBjb2xvcjogI0ZGRjtcbiAgICBkaXNwbGF5OiBub25lO1xuICAgIHBvc2l0aW9uOiBhYnNvbHV0ZTtcbiAgICB3aGl0ZS1zcGFjZTogbm93cmFwO1xuICAgIHotaW5kZXg6IDE7XG59XG5cbi54dGVybSAuY29tcG9zaXRpb24tdmlldy5hY3RpdmUge1xuICAgIGRpc3BsYXk6IGJsb2NrO1xufVxuXG4ueHRlcm0gLnh0ZXJtLXZpZXdwb3J0IHtcbiAgICAvKiBPbiBPUyBYIHRoaXMgaXMgcmVxdWlyZWQgaW4gb3JkZXIgZm9yIHRoZSBzY3JvbGwgYmFyIHRvIGFwcGVhciBmdWxseSBvcGFxdWUgKi9cbiAgICBiYWNrZ3JvdW5kLWNvbG9yOiAjMDAwO1xuICAgIG92ZXJmbG93LXk6IHNjcm9sbDtcbiAgICBjdXJzb3I6IGRlZmF1bHQ7XG4gICAgcG9zaXRpb246IGFic29sdXRlO1xuICAgIHJpZ2h0OiAwO1xuICAgIGxlZnQ6IDA7XG4gICAgdG9wOiAwO1xuICAgIGJvdHRvbTogMDtcbn1cblxuLnh0ZXJtIC54dGVybS1zY3JlZW4ge1xuICAgIHBvc2l0aW9uOiByZWxhdGl2ZTtcbn1cblxuLnh0ZXJtIC54dGVybS1zY3JlZW4gY2FudmFzIHtcbiAgICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gICAgbGVmdDogMDtcbiAgICB0b3A6IDA7XG59XG5cbi54dGVybSAueHRlcm0tc2Nyb2xsLWFyZWEge1xuICAgIHZpc2liaWxpdHk6IGhpZGRlbjtcbn1cblxuLnh0ZXJtLWNoYXItbWVhc3VyZS1lbGVtZW50IHtcbiAgICBkaXNwbGF5OiBpbmxpbmUtYmxvY2s7XG4gICAgdmlzaWJpbGl0eTogaGlkZGVuO1xuICAgIHBvc2l0aW9uOiBhYnNvbHV0ZTtcbiAgICB0b3A6IDA7XG4gICAgbGVmdDogLTk5OTllbTtcbiAgICBsaW5lLWhlaWdodDogbm9ybWFsO1xufVxuXG4ueHRlcm0ge1xuICAgIGN1cnNvcjogdGV4dDtcbn1cblxuLnh0ZXJtLmVuYWJsZS1tb3VzZS1ldmVudHMge1xuICAgIC8qIFdoZW4gbW91c2UgZXZlbnRzIGFyZSBlbmFibGVkIChlZy4gdG11eCksIHJldmVydCB0byB0aGUgc3RhbmRhcmQgcG9pbnRlciBjdXJzb3IgKi9cbiAgICBjdXJzb3I6IGRlZmF1bHQ7XG59XG5cbi54dGVybS54dGVybS1jdXJzb3ItcG9pbnRlcixcbi54dGVybSAueHRlcm0tY3Vyc29yLXBvaW50ZXIge1xuICAgIGN1cnNvcjogcG9pbnRlcjtcbn1cblxuLnh0ZXJtLmNvbHVtbi1zZWxlY3QuZm9jdXMge1xuICAgIC8qIENvbHVtbiBzZWxlY3Rpb24gbW9kZSAqL1xuICAgIGN1cnNvcjogY3Jvc3NoYWlyO1xufVxuXG4ueHRlcm0gLnh0ZXJtLWFjY2Vzc2liaWxpdHksXG4ueHRlcm0gLnh0ZXJtLW1lc3NhZ2Uge1xuICAgIHBvc2l0aW9uOiBhYnNvbHV0ZTtcbiAgICBsZWZ0OiAwO1xuICAgIHRvcDogMDtcbiAgICBib3R0b206IDA7XG4gICAgcmlnaHQ6IDA7XG4gICAgei1pbmRleDogMTA7XG4gICAgY29sb3I6IHRyYW5zcGFyZW50O1xufVxuXG4ueHRlcm0gLmxpdmUtcmVnaW9uIHtcbiAgICBwb3NpdGlvbjogYWJzb2x1dGU7XG4gICAgbGVmdDogLTk5OTlweDtcbiAgICB3aWR0aDogMXB4O1xuICAgIGhlaWdodDogMXB4O1xuICAgIG92ZXJmbG93OiBoaWRkZW47XG59XG5cbi54dGVybS1kaW0ge1xuICAgIG9wYWNpdHk6IDAuNTtcbn1cblxuLnh0ZXJtLXVuZGVybGluZSB7XG4gICAgdGV4dC1kZWNvcmF0aW9uOiB1bmRlcmxpbmU7XG59XG5cbi54dGVybS1zdHJpa2V0aHJvdWdoIHtcbiAgICB0ZXh0LWRlY29yYXRpb246IGxpbmUtdGhyb3VnaDtcbn1cbicsIiJdKTtjb25zdCBhPXN9LDY0NTplPT57InVzZSBzdHJpY3QiO2UuZXhwb3J0cz1mdW5jdGlvbihlKXt2YXIgdD1bXTtyZXR1cm4gdC50b1N0cmluZz1mdW5jdGlvbigpe3JldHVybiB0aGlzLm1hcCgoZnVuY3Rpb24odCl7dmFyIHI9IiIsaT12b2lkIDAhPT10WzVdO3JldHVybiB0WzRdJiYocis9IkBzdXBwb3J0cyAoIi5jb25jYXQodFs0XSwiKSB7IikpLHRbMl0mJihyKz0iQG1lZGlhICIuY29uY2F0KHRbMl0sIiB7IikpLGkmJihyKz0iQGxheWVyIi5jb25jYXQodFs1XS5sZW5ndGg+MD8iICIuY29uY2F0KHRbNV0pOiIiLCIgeyIpKSxyKz1lKHQpLGkmJihyKz0ifSIpLHRbMl0mJihyKz0ifSIpLHRbNF0mJihyKz0ifSIpLHJ9KSkuam9pbigiIil9LHQuaT1mdW5jdGlvbihlLHIsaSxuLG8peyJzdHJpbmciPT10eXBlb2YgZSYmKGU9W1tudWxsLGUsdm9pZCAwXV0pO3ZhciBzPXt9O2lmKGkpZm9yKHZhciBhPTA7YTx0aGlzLmxlbmd0aDthKyspe3ZhciBjPXRoaXNbYV1bMF07bnVsbCE9YyYmKHNbY109ITApfWZvcih2YXIgbD0wO2w8ZS5sZW5ndGg7bCsrKXt2YXIgdT1bXS5jb25jYXQoZVtsXSk7aSYmc1t1WzBdXXx8KHZvaWQgMCE9PW8mJih2b2lkIDA9PT11WzVdfHwodVsxXT0iQGxheWVyIi5jb25jYXQodVs1XS5sZW5ndGg+MD8iICIuY29uY2F0KHVbNV0pOiIiLCIgeyIpLmNvbmNhdCh1WzFdLCJ9IikpLHVbNV09byksciYmKHVbMl0/KHVbMV09IkBtZWRpYSAiLmNvbmNhdCh1WzJdLCIgeyIpLmNvbmNhdCh1WzFdLCJ9IiksdVsyXT1yKTp1WzJdPXIpLG4mJih1WzRdPyh1WzFdPSJAc3VwcG9ydHMgKCIuY29uY2F0KHVbNF0sIikgeyIpLmNvbmNhdCh1WzFdLCJ9IiksdVs0XT1uKTp1WzRdPSIiLmNvbmNhdChuKSksdC5wdXNoKHUpKX19LHR9fSw4MTplPT57InVzZSBzdHJpY3QiO2UuZXhwb3J0cz1mdW5jdGlvbihlKXtyZXR1cm4gZVsxXX19LDQ4NjpmdW5jdGlvbihlLHQscil7dmFyIGk7ZT1yLm5tZChlKSxmdW5jdGlvbigpe3ZhciBuLG89IkV4cGVjdGVkIGEgZnVuY3Rpb24iLHM9Il9fbG9kYXNoX2hhc2hfdW5kZWZpbmVkX18iLGE9Il9fbG9kYXNoX3BsYWNlaG9sZGVyX18iLGM9MzIsbD0xMjgsdT0xLzAsaD05MDA3MTk5MjU0NzQwOTkxLGY9TmFOLF89NDI5NDk2NzI5NSxkPVtbImFyeSIsbF0sWyJiaW5kIiwxXSxbImJpbmRLZXkiLDJdLFsiY3VycnkiLDhdLFsiY3VycnlSaWdodCIsMTZdLFsiZmxpcCIsNTEyXSxbInBhcnRpYWwiLGNdLFsicGFydGlhbFJpZ2h0Iiw2NF0sWyJyZWFyZyIsMjU2XV0scD0iW29iamVjdCBBcmd1bWVudHNdIix2PSJbb2JqZWN0IEFycmF5XSIsZz0iW29iamVjdCBCb29sZWFuXSIseT0iW29iamVjdCBEYXRlXSIsbT0iW29iamVjdCBFcnJvcl0iLGI9IltvYmplY3QgRnVuY3Rpb25dIixTPSJbb2JqZWN0IEdlbmVyYXRvckZ1bmN0aW9uXSIsQz0iW29iamVjdCBNYXBdIix3PSJbb2JqZWN0IE51bWJlcl0iLEw9IltvYmplY3QgT2JqZWN0XSIsRT0iW29iamVjdCBQcm9taXNlXSIseD0iW29iamVjdCBSZWdFeHBdIixBPSJbb2JqZWN0IFNldF0iLGs9IltvYmplY3QgU3RyaW5nXSIsTT0iW29iamVjdCBTeW1ib2xdIixSPSJbb2JqZWN0IFdlYWtNYXBdIixUPSJbb2JqZWN0IEFycmF5QnVmZmVyXSIsTz0iW29iamVjdCBEYXRhVmlld10iLEI9IltvYmplY3QgRmxvYXQzMkFycmF5XSIsRD0iW29iamVjdCBGbG9hdDY0QXJyYXldIixQPSJbb2JqZWN0IEludDhBcnJheV0iLEk9IltvYmplY3QgSW50MTZBcnJheV0iLEg9IltvYmplY3QgSW50MzJBcnJheV0iLGo9IltvYmplY3QgVWludDhBcnJheV0iLEY9IltvYmplY3QgVWludDhDbGFtcGVkQXJyYXldIixXPSJbb2JqZWN0IFVpbnQxNkFycmF5XSIsVT0iW29iamVjdCBVaW50MzJBcnJheV0iLHE9L1xiX19wIFwrPSAnJzsvZyxOPS9cYihfX3AgXCs9KSAnJyBcKy9nLHo9LyhfX2VcKC4qP1wpfFxiX190XCkpIFwrXG4nJzsvZyxLPS8mKD86YW1wfGx0fGd0fHF1b3R8IzM5KTsvZyxWPS9bJjw+IiddL2csRz1SZWdFeHAoSy5zb3VyY2UpLFk9UmVnRXhwKFYuc291cmNlKSxYPS88JS0oW1xzXFNdKz8pJT4vZyxaPS88JShbXHNcU10rPyklPi9nLEo9LzwlPShbXHNcU10rPyklPi9nLCQ9L1wufFxbKD86W15bXF1dKnwoWyInXSkoPzooPyFcMSlbXlxcXXxcXC4pKj9cMSlcXS8sUT0vXlx3KiQvLGVlPS9bXi5bXF1dK3xcWyg/OigtP1xkKyg/OlwuXGQrKT8pfChbIiddKSgoPzooPyFcMilbXlxcXXxcXC4pKj8pXDIpXF18KD89KD86XC58XFtcXSkoPzpcLnxcW1xdfCQpKS9nLHRlPS9bXFxeJC4qKz8oKVtcXXt9fF0vZyxyZT1SZWdFeHAodGUuc291cmNlKSxpZT0vXlxzKy8sbmU9L1xzLyxvZT0vXHsoPzpcblwvXCogXFt3cmFwcGVkIHdpdGggLitcXSBcKlwvKT9cbj8vLHNlPS9ce1xuXC9cKiBcW3dyYXBwZWQgd2l0aCAoLispXF0gXCovLGFlPS8sPyAmIC8sY2U9L1teXHgwMC1ceDJmXHgzYS1ceDQwXHg1Yi1ceDYwXHg3Yi1ceDdmXSsvZyxsZT0vWygpPSx7fVxbXF1cL1xzXS8sdWU9L1xcKFxcKT8vZyxoZT0vXCRceyhbXlxcfV0qKD86XFwuW15cXH1dKikqKVx9L2csZmU9L1x3KiQvLF9lPS9eWy0rXTB4WzAtOWEtZl0rJC9pLGRlPS9eMGJbMDFdKyQvaSxwZT0vXlxbb2JqZWN0IC4rP0NvbnN0cnVjdG9yXF0kLyx2ZT0vXjBvWzAtN10rJC9pLGdlPS9eKD86MHxbMS05XVxkKikkLyx5ZT0vW1x4YzAtXHhkNlx4ZDgtXHhmNlx4ZjgtXHhmZlx1MDEwMC1cdTAxN2ZdL2csbWU9LygkXikvLGJlPS9bJ1xuXHJcdTIwMjhcdTIwMjlcXF0vZyxTZT0iXFx1MDMwMC1cXHUwMzZmXFx1ZmUyMC1cXHVmZTJmXFx1MjBkMC1cXHUyMGZmIixDZT0iYS16XFx4ZGYtXFx4ZjZcXHhmOC1cXHhmZiIsd2U9IkEtWlxceGMwLVxceGQ2XFx4ZDgtXFx4ZGUiLExlPSJcXHhhY1xceGIxXFx4ZDdcXHhmN1xceDAwLVxceDJmXFx4M2EtXFx4NDBcXHg1Yi1cXHg2MFxceDdiLVxceGJmXFx1MjAwMC1cXHUyMDZmIFxcdFxceDBiXFxmXFx4YTBcXHVmZWZmXFxuXFxyXFx1MjAyOFxcdTIwMjlcXHUxNjgwXFx1MTgwZVxcdTIwMDBcXHUyMDAxXFx1MjAwMlxcdTIwMDNcXHUyMDA0XFx1MjAwNVxcdTIwMDZcXHUyMDA3XFx1MjAwOFxcdTIwMDlcXHUyMDBhXFx1MjAyZlxcdTIwNWZcXHUzMDAwIixFZT0iWyIrTGUrIl0iLHhlPSJbIitTZSsiXSIsQWU9IlxcZCsiLGtlPSJbIitDZSsiXSIsTWU9IlteXFx1ZDgwMC1cXHVkZmZmIitMZStBZSsiXFx1MjcwMC1cXHUyN2JmIitDZSt3ZSsiXSIsUmU9IlxcdWQ4M2NbXFx1ZGZmYi1cXHVkZmZmXSIsVGU9IlteXFx1ZDgwMC1cXHVkZmZmXSIsT2U9Iig/OlxcdWQ4M2NbXFx1ZGRlNi1cXHVkZGZmXSl7Mn0iLEJlPSJbXFx1ZDgwMC1cXHVkYmZmXVtcXHVkYzAwLVxcdWRmZmZdIixEZT0iWyIrd2UrIl0iLFBlPSIoPzoiK2tlKyJ8IitNZSsiKSIsSWU9Iig/OiIrRGUrInwiK01lKyIpIixIZT0iKD86WyfigJldKD86ZHxsbHxtfHJlfHN8dHx2ZSkpPyIsamU9Iig/Olsn4oCZXSg/OkR8TEx8TXxSRXxTfFR8VkUpKT8iLEZlPSIoPzoiK3hlKyJ8IitSZSsiKT8iLFdlPSJbXFx1ZmUwZVxcdWZlMGZdPyIsVWU9V2UrRmUrIig/OlxcdTIwMGQoPzoiK1tUZSxPZSxCZV0uam9pbigifCIpKyIpIitXZStGZSsiKSoiLHFlPSIoPzoiK1siW1xcdTI3MDAtXFx1MjdiZl0iLE9lLEJlXS5qb2luKCJ8IikrIikiK1VlLE5lPSIoPzoiK1tUZSt4ZSsiPyIseGUsT2UsQmUsIltcXHVkODAwLVxcdWRmZmZdIl0uam9pbigifCIpKyIpIix6ZT1SZWdFeHAoIlsn4oCZXSIsImciKSxLZT1SZWdFeHAoeGUsImciKSxWZT1SZWdFeHAoUmUrIig/PSIrUmUrIil8IitOZStVZSwiZyIpLEdlPVJlZ0V4cChbRGUrIj8iK2tlKyIrIitIZSsiKD89IitbRWUsRGUsIiQiXS5qb2luKCJ8IikrIikiLEllKyIrIitqZSsiKD89IitbRWUsRGUrUGUsIiQiXS5qb2luKCJ8IikrIikiLERlKyI/IitQZSsiKyIrSGUsRGUrIisiK2plLCJcXGQqKD86MVNUfDJORHwzUkR8KD8hWzEyM10pXFxkVEgpKD89XFxifFthLXpfXSkiLCJcXGQqKD86MXN0fDJuZHwzcmR8KD8hWzEyM10pXFxkdGgpKD89XFxifFtBLVpfXSkiLEFlLHFlXS5qb2luKCJ8IiksImciKSxZZT1SZWdFeHAoIltcXHUyMDBkXFx1ZDgwMC1cXHVkZmZmIitTZSsiXFx1ZmUwZVxcdWZlMGZdIiksWGU9L1thLXpdW0EtWl18W0EtWl17Mn1bYS16XXxbMC05XVthLXpBLVpdfFthLXpBLVpdWzAtOV18W15hLXpBLVowLTkgXS8sWmU9WyJBcnJheSIsIkJ1ZmZlciIsIkRhdGFWaWV3IiwiRGF0ZSIsIkVycm9yIiwiRmxvYXQzMkFycmF5IiwiRmxvYXQ2NEFycmF5IiwiRnVuY3Rpb24iLCJJbnQ4QXJyYXkiLCJJbnQxNkFycmF5IiwiSW50MzJBcnJheSIsIk1hcCIsIk1hdGgiLCJPYmplY3QiLCJQcm9taXNlIiwiUmVnRXhwIiwiU2V0IiwiU3RyaW5nIiwiU3ltYm9sIiwiVHlwZUVycm9yIiwiVWludDhBcnJheSIsIlVpbnQ4Q2xhbXBlZEFycmF5IiwiVWludDE2QXJyYXkiLCJVaW50MzJBcnJheSIsIldlYWtNYXAiLCJfIiwiY2xlYXJUaW1lb3V0IiwiaXNGaW5pdGUiLCJwYXJzZUludCIsInNldFRpbWVvdXQiXSxKZT0tMSwkZT17fTskZVtCXT0kZVtEXT0kZVtQXT0kZVtJXT0kZVtIXT0kZVtqXT0kZVtGXT0kZVtXXT0kZVtVXT0hMCwkZVtwXT0kZVt2XT0kZVtUXT0kZVtnXT0kZVtPXT0kZVt5XT0kZVttXT0kZVtiXT0kZVtDXT0kZVt3XT0kZVtMXT0kZVt4XT0kZVtBXT0kZVtrXT0kZVtSXT0hMTt2YXIgUWU9e307UWVbcF09UWVbdl09UWVbVF09UWVbT109UWVbZ109UWVbeV09UWVbQl09UWVbRF09UWVbUF09UWVbSV09UWVbSF09UWVbQ109UWVbd109UWVbTF09UWVbeF09UWVbQV09UWVba109UWVbTV09UWVbal09UWVbRl09UWVbV109UWVbVV09ITAsUWVbbV09UWVbYl09UWVbUl09ITE7dmFyIGV0PXsiXFwiOiJcXCIsIiciOiInIiwiXG4iOiJuIiwiXHIiOiJyIiwiXHUyMDI4IjoidTIwMjgiLCJcdTIwMjkiOiJ1MjAyOSJ9LHR0PXBhcnNlRmxvYXQscnQ9cGFyc2VJbnQsaXQ9Im9iamVjdCI9PXR5cGVvZiByLmcmJnIuZyYmci5nLk9iamVjdD09PU9iamVjdCYmci5nLG50PSJvYmplY3QiPT10eXBlb2Ygc2VsZiYmc2VsZiYmc2VsZi5PYmplY3Q9PT1PYmplY3QmJnNlbGYsb3Q9aXR8fG50fHxGdW5jdGlvbigicmV0dXJuIHRoaXMiKSgpLHN0PXQmJiF0Lm5vZGVUeXBlJiZ0LGF0PXN0JiZlJiYhZS5ub2RlVHlwZSYmZSxjdD1hdCYmYXQuZXhwb3J0cz09PXN0LGx0PWN0JiZpdC5wcm9jZXNzLHV0PWZ1bmN0aW9uKCl7dHJ5e3JldHVybiBhdCYmYXQucmVxdWlyZSYmYXQucmVxdWlyZSgidXRpbCIpLnR5cGVzfHxsdCYmbHQuYmluZGluZyYmbHQuYmluZGluZygidXRpbCIpfWNhdGNoKGUpe319KCksaHQ9dXQmJnV0LmlzQXJyYXlCdWZmZXIsZnQ9dXQmJnV0LmlzRGF0ZSxfdD11dCYmdXQuaXNNYXAsZHQ9dXQmJnV0LmlzUmVnRXhwLHB0PXV0JiZ1dC5pc1NldCx2dD11dCYmdXQuaXNUeXBlZEFycmF5O2Z1bmN0aW9uIGd0KGUsdCxyKXtzd2l0Y2goci5sZW5ndGgpe2Nhc2UgMDpyZXR1cm4gZS5jYWxsKHQpO2Nhc2UgMTpyZXR1cm4gZS5jYWxsKHQsclswXSk7Y2FzZSAyOnJldHVybiBlLmNhbGwodCxyWzBdLHJbMV0pO2Nhc2UgMzpyZXR1cm4gZS5jYWxsKHQsclswXSxyWzFdLHJbMl0pfXJldHVybiBlLmFwcGx5KHQscil9ZnVuY3Rpb24geXQoZSx0LHIsaSl7Zm9yKHZhciBuPS0xLG89bnVsbD09ZT8wOmUubGVuZ3RoOysrbjxvOyl7dmFyIHM9ZVtuXTt0KGkscyxyKHMpLGUpfXJldHVybiBpfWZ1bmN0aW9uIG10KGUsdCl7Zm9yKHZhciByPS0xLGk9bnVsbD09ZT8wOmUubGVuZ3RoOysrcjxpJiYhMSE9PXQoZVtyXSxyLGUpOyk7cmV0dXJuIGV9ZnVuY3Rpb24gYnQoZSx0KXtmb3IodmFyIHI9bnVsbD09ZT8wOmUubGVuZ3RoO3ItLSYmITEhPT10KGVbcl0scixlKTspO3JldHVybiBlfWZ1bmN0aW9uIFN0KGUsdCl7Zm9yKHZhciByPS0xLGk9bnVsbD09ZT8wOmUubGVuZ3RoOysrcjxpOylpZighdChlW3JdLHIsZSkpcmV0dXJuITE7cmV0dXJuITB9ZnVuY3Rpb24gQ3QoZSx0KXtmb3IodmFyIHI9LTEsaT1udWxsPT1lPzA6ZS5sZW5ndGgsbj0wLG89W107KytyPGk7KXt2YXIgcz1lW3JdO3QocyxyLGUpJiYob1tuKytdPXMpfXJldHVybiBvfWZ1bmN0aW9uIHd0KGUsdCl7cmV0dXJuIShudWxsPT1lfHwhZS5sZW5ndGgpJiZCdChlLHQsMCk+LTF9ZnVuY3Rpb24gTHQoZSx0LHIpe2Zvcih2YXIgaT0tMSxuPW51bGw9PWU/MDplLmxlbmd0aDsrK2k8bjspaWYocih0LGVbaV0pKXJldHVybiEwO3JldHVybiExfWZ1bmN0aW9uIEV0KGUsdCl7Zm9yKHZhciByPS0xLGk9bnVsbD09ZT8wOmUubGVuZ3RoLG49QXJyYXkoaSk7KytyPGk7KW5bcl09dChlW3JdLHIsZSk7cmV0dXJuIG59ZnVuY3Rpb24geHQoZSx0KXtmb3IodmFyIHI9LTEsaT10Lmxlbmd0aCxuPWUubGVuZ3RoOysrcjxpOyllW24rcl09dFtyXTtyZXR1cm4gZX1mdW5jdGlvbiBBdChlLHQscixpKXt2YXIgbj0tMSxvPW51bGw9PWU/MDplLmxlbmd0aDtmb3IoaSYmbyYmKHI9ZVsrK25dKTsrK248bzspcj10KHIsZVtuXSxuLGUpO3JldHVybiByfWZ1bmN0aW9uIGt0KGUsdCxyLGkpe3ZhciBuPW51bGw9PWU/MDplLmxlbmd0aDtmb3IoaSYmbiYmKHI9ZVstLW5dKTtuLS07KXI9dChyLGVbbl0sbixlKTtyZXR1cm4gcn1mdW5jdGlvbiBNdChlLHQpe2Zvcih2YXIgcj0tMSxpPW51bGw9PWU/MDplLmxlbmd0aDsrK3I8aTspaWYodChlW3JdLHIsZSkpcmV0dXJuITA7cmV0dXJuITF9dmFyIFJ0PUh0KCJsZW5ndGgiKTtmdW5jdGlvbiBUdChlLHQscil7dmFyIGk7cmV0dXJuIHIoZSwoZnVuY3Rpb24oZSxyLG4pe2lmKHQoZSxyLG4pKXJldHVybiBpPXIsITF9KSksaX1mdW5jdGlvbiBPdChlLHQscixpKXtmb3IodmFyIG49ZS5sZW5ndGgsbz1yKyhpPzE6LTEpO2k/by0tOisrbzxuOylpZih0KGVbb10sbyxlKSlyZXR1cm4gbztyZXR1cm4tMX1mdW5jdGlvbiBCdChlLHQscil7cmV0dXJuIHQ9PXQ/ZnVuY3Rpb24oZSx0LHIpe2Zvcih2YXIgaT1yLTEsbj1lLmxlbmd0aDsrK2k8bjspaWYoZVtpXT09PXQpcmV0dXJuIGk7cmV0dXJuLTF9KGUsdCxyKTpPdChlLFB0LHIpfWZ1bmN0aW9uIER0KGUsdCxyLGkpe2Zvcih2YXIgbj1yLTEsbz1lLmxlbmd0aDsrK248bzspaWYoaShlW25dLHQpKXJldHVybiBuO3JldHVybi0xfWZ1bmN0aW9uIFB0KGUpe3JldHVybiBlIT1lfWZ1bmN0aW9uIEl0KGUsdCl7dmFyIHI9bnVsbD09ZT8wOmUubGVuZ3RoO3JldHVybiByP1d0KGUsdCkvcjpmfWZ1bmN0aW9uIEh0KGUpe3JldHVybiBmdW5jdGlvbih0KXtyZXR1cm4gbnVsbD09dD9uOnRbZV19fWZ1bmN0aW9uIGp0KGUpe3JldHVybiBmdW5jdGlvbih0KXtyZXR1cm4gbnVsbD09ZT9uOmVbdF19fWZ1bmN0aW9uIEZ0KGUsdCxyLGksbil7cmV0dXJuIG4oZSwoZnVuY3Rpb24oZSxuLG8pe3I9aT8oaT0hMSxlKTp0KHIsZSxuLG8pfSkpLHJ9ZnVuY3Rpb24gV3QoZSx0KXtmb3IodmFyIHIsaT0tMSxvPWUubGVuZ3RoOysraTxvOyl7dmFyIHM9dChlW2ldKTtzIT09biYmKHI9cj09PW4/czpyK3MpfXJldHVybiByfWZ1bmN0aW9uIFV0KGUsdCl7Zm9yKHZhciByPS0xLGk9QXJyYXkoZSk7KytyPGU7KWlbcl09dChyKTtyZXR1cm4gaX1mdW5jdGlvbiBxdChlKXtyZXR1cm4gZT9lLnNsaWNlKDAsc3IoZSkrMSkucmVwbGFjZShpZSwiIik6ZX1mdW5jdGlvbiBOdChlKXtyZXR1cm4gZnVuY3Rpb24odCl7cmV0dXJuIGUodCl9fWZ1bmN0aW9uIHp0KGUsdCl7cmV0dXJuIEV0KHQsKGZ1bmN0aW9uKHQpe3JldHVybiBlW3RdfSkpfWZ1bmN0aW9uIEt0KGUsdCl7cmV0dXJuIGUuaGFzKHQpfWZ1bmN0aW9uIFZ0KGUsdCl7Zm9yKHZhciByPS0xLGk9ZS5sZW5ndGg7KytyPGkmJkJ0KHQsZVtyXSwwKT4tMTspO3JldHVybiByfWZ1bmN0aW9uIEd0KGUsdCl7Zm9yKHZhciByPWUubGVuZ3RoO3ItLSYmQnQodCxlW3JdLDApPi0xOyk7cmV0dXJuIHJ9ZnVuY3Rpb24gWXQoZSx0KXtmb3IodmFyIHI9ZS5sZW5ndGgsaT0wO3ItLTspZVtyXT09PXQmJisraTtyZXR1cm4gaX12YXIgWHQ9anQoe8OAOiJBIizDgToiQSIsw4I6IkEiLMODOiJBIizDhDoiQSIsw4U6IkEiLMOgOiJhIizDoToiYSIsw6I6ImEiLMOjOiJhIizDpDoiYSIsw6U6ImEiLMOHOiJDIizDpzoiYyIsw5A6IkQiLMOwOiJkIizDiDoiRSIsw4k6IkUiLMOKOiJFIizDizoiRSIsw6g6ImUiLMOpOiJlIizDqjoiZSIsw6s6ImUiLMOMOiJJIizDjToiSSIsw446IkkiLMOPOiJJIizDrDoiaSIsw606ImkiLMOuOiJpIizDrzoiaSIsw5E6Ik4iLMOxOiJuIizDkjoiTyIsw5M6Ik8iLMOUOiJPIizDlToiTyIsw5Y6Ik8iLMOYOiJPIizDsjoibyIsw7M6Im8iLMO0OiJvIizDtToibyIsw7Y6Im8iLMO4OiJvIizDmToiVSIsw5o6IlUiLMObOiJVIizDnDoiVSIsw7k6InUiLMO6OiJ1IizDuzoidSIsw7w6InUiLMOdOiJZIizDvToieSIsw786InkiLMOGOiJBZSIsw6Y6ImFlIizDnjoiVGgiLMO+OiJ0aCIsw586InNzIizEgDoiQSIsxII6IkEiLMSEOiJBIizEgToiYSIsxIM6ImEiLMSFOiJhIizEhjoiQyIsxIg6IkMiLMSKOiJDIizEjDoiQyIsxIc6ImMiLMSJOiJjIizEizoiYyIsxI06ImMiLMSOOiJEIizEkDoiRCIsxI86ImQiLMSROiJkIizEkjoiRSIsxJQ6IkUiLMSWOiJFIizEmDoiRSIsxJo6IkUiLMSTOiJlIizElToiZSIsxJc6ImUiLMSZOiJlIizEmzoiZSIsxJw6IkciLMSeOiJHIizEoDoiRyIsxKI6IkciLMSdOiJnIizEnzoiZyIsxKE6ImciLMSjOiJnIizEpDoiSCIsxKY6IkgiLMSlOiJoIizEpzoiaCIsxKg6IkkiLMSqOiJJIizErDoiSSIsxK46IkkiLMSwOiJJIizEqToiaSIsxKs6ImkiLMStOiJpIizErzoiaSIsxLE6ImkiLMS0OiJKIizEtToiaiIsxLY6IksiLMS3OiJrIizEuDoiayIsxLk6IkwiLMS7OiJMIizEvToiTCIsxL86IkwiLMWBOiJMIizEujoibCIsxLw6ImwiLMS+OiJsIizFgDoibCIsxYI6ImwiLMWDOiJOIizFhToiTiIsxYc6Ik4iLMWKOiJOIizFhDoibiIsxYY6Im4iLMWIOiJuIizFizoibiIsxYw6Ik8iLMWOOiJPIizFkDoiTyIsxY06Im8iLMWPOiJvIizFkToibyIsxZQ6IlIiLMWWOiJSIizFmDoiUiIsxZU6InIiLMWXOiJyIizFmToiciIsxZo6IlMiLMWcOiJTIizFnjoiUyIsxaA6IlMiLMWbOiJzIizFnToicyIsxZ86InMiLMWhOiJzIizFojoiVCIsxaQ6IlQiLMWmOiJUIizFozoidCIsxaU6InQiLMWnOiJ0IizFqDoiVSIsxao6IlUiLMWsOiJVIizFrjoiVSIsxbA6IlUiLMWyOiJVIizFqToidSIsxas6InUiLMWtOiJ1IizFrzoidSIsxbE6InUiLMWzOiJ1IizFtDoiVyIsxbU6InciLMW2OiJZIizFtzoieSIsxbg6IlkiLMW5OiJaIizFuzoiWiIsxb06IloiLMW6OiJ6IizFvDoieiIsxb46InoiLMSyOiJJSiIsxLM6ImlqIizFkjoiT2UiLMWTOiJvZSIsxYk6IiduIizFvzoicyJ9KSxadD1qdCh7IiYiOiImYW1wOyIsIjwiOiImbHQ7IiwiPiI6IiZndDsiLCciJzoiJnF1b3Q7IiwiJyI6IiYjMzk7In0pO2Z1bmN0aW9uIEp0KGUpe3JldHVybiJcXCIrZXRbZV19ZnVuY3Rpb24gJHQoZSl7cmV0dXJuIFllLnRlc3QoZSl9ZnVuY3Rpb24gUXQoZSl7dmFyIHQ9LTEscj1BcnJheShlLnNpemUpO3JldHVybiBlLmZvckVhY2goKGZ1bmN0aW9uKGUsaSl7clsrK3RdPVtpLGVdfSkpLHJ9ZnVuY3Rpb24gZXIoZSx0KXtyZXR1cm4gZnVuY3Rpb24ocil7cmV0dXJuIGUodChyKSl9fWZ1bmN0aW9uIHRyKGUsdCl7Zm9yKHZhciByPS0xLGk9ZS5sZW5ndGgsbj0wLG89W107KytyPGk7KXt2YXIgcz1lW3JdO3MhPT10JiZzIT09YXx8KGVbcl09YSxvW24rK109cil9cmV0dXJuIG99ZnVuY3Rpb24gcnIoZSl7dmFyIHQ9LTEscj1BcnJheShlLnNpemUpO3JldHVybiBlLmZvckVhY2goKGZ1bmN0aW9uKGUpe3JbKyt0XT1lfSkpLHJ9ZnVuY3Rpb24gaXIoZSl7dmFyIHQ9LTEscj1BcnJheShlLnNpemUpO3JldHVybiBlLmZvckVhY2goKGZ1bmN0aW9uKGUpe3JbKyt0XT1bZSxlXX0pKSxyfWZ1bmN0aW9uIG5yKGUpe3JldHVybiAkdChlKT9mdW5jdGlvbihlKXtmb3IodmFyIHQ9VmUubGFzdEluZGV4PTA7VmUudGVzdChlKTspKyt0O3JldHVybiB0fShlKTpSdChlKX1mdW5jdGlvbiBvcihlKXtyZXR1cm4gJHQoZSk/ZnVuY3Rpb24oZSl7cmV0dXJuIGUubWF0Y2goVmUpfHxbXX0oZSk6ZnVuY3Rpb24oZSl7cmV0dXJuIGUuc3BsaXQoIiIpfShlKX1mdW5jdGlvbiBzcihlKXtmb3IodmFyIHQ9ZS5sZW5ndGg7dC0tJiZuZS50ZXN0KGUuY2hhckF0KHQpKTspO3JldHVybiB0fXZhciBhcj1qdCh7IiZhbXA7IjoiJiIsIiZsdDsiOiI8IiwiJmd0OyI6Ij4iLCImcXVvdDsiOiciJywiJiMzOTsiOiInIn0pLGNyPWZ1bmN0aW9uIGUodCl7dmFyIHIsaT0odD1udWxsPT10P290OmNyLmRlZmF1bHRzKG90Lk9iamVjdCgpLHQsY3IucGljayhvdCxaZSkpKS5BcnJheSxuZT10LkRhdGUsU2U9dC5FcnJvcixDZT10LkZ1bmN0aW9uLHdlPXQuTWF0aCxMZT10Lk9iamVjdCxFZT10LlJlZ0V4cCx4ZT10LlN0cmluZyxBZT10LlR5cGVFcnJvcixrZT1pLnByb3RvdHlwZSxNZT1DZS5wcm90b3R5cGUsUmU9TGUucHJvdG90eXBlLFRlPXRbIl9fY29yZS1qc19zaGFyZWRfXyJdLE9lPU1lLnRvU3RyaW5nLEJlPVJlLmhhc093blByb3BlcnR5LERlPTAsUGU9KHI9L1teLl0rJC8uZXhlYyhUZSYmVGUua2V5cyYmVGUua2V5cy5JRV9QUk9UT3x8IiIpKT8iU3ltYm9sKHNyYylfMS4iK3I6IiIsSWU9UmUudG9TdHJpbmcsSGU9T2UuY2FsbChMZSksamU9b3QuXyxGZT1FZSgiXiIrT2UuY2FsbChCZSkucmVwbGFjZSh0ZSwiXFwkJiIpLnJlcGxhY2UoL2hhc093blByb3BlcnR5fChmdW5jdGlvbikuKj8oPz1cXFwoKXwgZm9yIC4rPyg/PVxcXF0pL2csIiQxLio/IikrIiQiKSxXZT1jdD90LkJ1ZmZlcjpuLFVlPXQuU3ltYm9sLHFlPXQuVWludDhBcnJheSxOZT1XZT9XZS5hbGxvY1Vuc2FmZTpuLFZlPWVyKExlLmdldFByb3RvdHlwZU9mLExlKSxZZT1MZS5jcmVhdGUsZXQ9UmUucHJvcGVydHlJc0VudW1lcmFibGUsaXQ9a2Uuc3BsaWNlLG50PVVlP1VlLmlzQ29uY2F0U3ByZWFkYWJsZTpuLHN0PVVlP1VlLml0ZXJhdG9yOm4sYXQ9VWU/VWUudG9TdHJpbmdUYWc6bixsdD1mdW5jdGlvbigpe3RyeXt2YXIgZT1sbyhMZSwiZGVmaW5lUHJvcGVydHkiKTtyZXR1cm4gZSh7fSwiIix7fSksZX1jYXRjaChlKXt9fSgpLHV0PXQuY2xlYXJUaW1lb3V0IT09b3QuY2xlYXJUaW1lb3V0JiZ0LmNsZWFyVGltZW91dCxSdD1uZSYmbmUubm93IT09b3QuRGF0ZS5ub3cmJm5lLm5vdyxqdD10LnNldFRpbWVvdXQhPT1vdC5zZXRUaW1lb3V0JiZ0LnNldFRpbWVvdXQsbHI9d2UuY2VpbCx1cj13ZS5mbG9vcixocj1MZS5nZXRPd25Qcm9wZXJ0eVN5bWJvbHMsZnI9V2U/V2UuaXNCdWZmZXI6bixfcj10LmlzRmluaXRlLGRyPWtlLmpvaW4scHI9ZXIoTGUua2V5cyxMZSksdnI9d2UubWF4LGdyPXdlLm1pbix5cj1uZS5ub3csbXI9dC5wYXJzZUludCxicj13ZS5yYW5kb20sU3I9a2UucmV2ZXJzZSxDcj1sbyh0LCJEYXRhVmlldyIpLHdyPWxvKHQsIk1hcCIpLExyPWxvKHQsIlByb21pc2UiKSxFcj1sbyh0LCJTZXQiKSx4cj1sbyh0LCJXZWFrTWFwIiksQXI9bG8oTGUsImNyZWF0ZSIpLGtyPXhyJiZuZXcgeHIsTXI9e30sUnI9Rm8oQ3IpLFRyPUZvKHdyKSxPcj1GbyhMciksQnI9Rm8oRXIpLERyPUZvKHhyKSxQcj1VZT9VZS5wcm90b3R5cGU6bixJcj1Qcj9Qci52YWx1ZU9mOm4sSHI9UHI/UHIudG9TdHJpbmc6bjtmdW5jdGlvbiBqcihlKXtpZihyYShlKSYmIUtzKGUpJiYhKGUgaW5zdGFuY2VvZiBxcikpe2lmKGUgaW5zdGFuY2VvZiBVcilyZXR1cm4gZTtpZihCZS5jYWxsKGUsIl9fd3JhcHBlZF9fIikpcmV0dXJuIFdvKGUpfXJldHVybiBuZXcgVXIoZSl9dmFyIEZyPWZ1bmN0aW9uKCl7ZnVuY3Rpb24gZSgpe31yZXR1cm4gZnVuY3Rpb24odCl7aWYoIXRhKHQpKXJldHVybnt9O2lmKFllKXJldHVybiBZZSh0KTtlLnByb3RvdHlwZT10O3ZhciByPW5ldyBlO3JldHVybiBlLnByb3RvdHlwZT1uLHJ9fSgpO2Z1bmN0aW9uIFdyKCl7fWZ1bmN0aW9uIFVyKGUsdCl7dGhpcy5fX3dyYXBwZWRfXz1lLHRoaXMuX19hY3Rpb25zX189W10sdGhpcy5fX2NoYWluX189ISF0LHRoaXMuX19pbmRleF9fPTAsdGhpcy5fX3ZhbHVlc19fPW59ZnVuY3Rpb24gcXIoZSl7dGhpcy5fX3dyYXBwZWRfXz1lLHRoaXMuX19hY3Rpb25zX189W10sdGhpcy5fX2Rpcl9fPTEsdGhpcy5fX2ZpbHRlcmVkX189ITEsdGhpcy5fX2l0ZXJhdGVlc19fPVtdLHRoaXMuX190YWtlQ291bnRfXz1fLHRoaXMuX192aWV3c19fPVtdfWZ1bmN0aW9uIE5yKGUpe3ZhciB0PS0xLHI9bnVsbD09ZT8wOmUubGVuZ3RoO2Zvcih0aGlzLmNsZWFyKCk7Kyt0PHI7KXt2YXIgaT1lW3RdO3RoaXMuc2V0KGlbMF0saVsxXSl9fWZ1bmN0aW9uIHpyKGUpe3ZhciB0PS0xLHI9bnVsbD09ZT8wOmUubGVuZ3RoO2Zvcih0aGlzLmNsZWFyKCk7Kyt0PHI7KXt2YXIgaT1lW3RdO3RoaXMuc2V0KGlbMF0saVsxXSl9fWZ1bmN0aW9uIEtyKGUpe3ZhciB0PS0xLHI9bnVsbD09ZT8wOmUubGVuZ3RoO2Zvcih0aGlzLmNsZWFyKCk7Kyt0PHI7KXt2YXIgaT1lW3RdO3RoaXMuc2V0KGlbMF0saVsxXSl9fWZ1bmN0aW9uIFZyKGUpe3ZhciB0PS0xLHI9bnVsbD09ZT8wOmUubGVuZ3RoO2Zvcih0aGlzLl9fZGF0YV9fPW5ldyBLcjsrK3Q8cjspdGhpcy5hZGQoZVt0XSl9ZnVuY3Rpb24gR3IoZSl7dmFyIHQ9dGhpcy5fX2RhdGFfXz1uZXcgenIoZSk7dGhpcy5zaXplPXQuc2l6ZX1mdW5jdGlvbiBZcihlLHQpe3ZhciByPUtzKGUpLGk9IXImJnpzKGUpLG49IXImJiFpJiZYcyhlKSxvPSFyJiYhaSYmIW4mJnVhKGUpLHM9cnx8aXx8bnx8byxhPXM/VXQoZS5sZW5ndGgseGUpOltdLGM9YS5sZW5ndGg7Zm9yKHZhciBsIGluIGUpIXQmJiFCZS5jYWxsKGUsbCl8fHMmJigibGVuZ3RoIj09bHx8biYmKCJvZmZzZXQiPT1sfHwicGFyZW50Ij09bCl8fG8mJigiYnVmZmVyIj09bHx8ImJ5dGVMZW5ndGgiPT1sfHwiYnl0ZU9mZnNldCI9PWwpfHxnbyhsLGMpKXx8YS5wdXNoKGwpO3JldHVybiBhfWZ1bmN0aW9uIFhyKGUpe3ZhciB0PWUubGVuZ3RoO3JldHVybiB0P2VbS2koMCx0LTEpXTpufWZ1bmN0aW9uIFpyKGUsdCl7cmV0dXJuIERvKEFuKGUpLG9pKHQsMCxlLmxlbmd0aCkpfWZ1bmN0aW9uIEpyKGUpe3JldHVybiBEbyhBbihlKSl9ZnVuY3Rpb24gJHIoZSx0LHIpeyhyIT09biYmIVVzKGVbdF0scil8fHI9PT1uJiYhKHQgaW4gZSkpJiZpaShlLHQscil9ZnVuY3Rpb24gUXIoZSx0LHIpe3ZhciBpPWVbdF07QmUuY2FsbChlLHQpJiZVcyhpLHIpJiYociE9PW58fHQgaW4gZSl8fGlpKGUsdCxyKX1mdW5jdGlvbiBlaShlLHQpe2Zvcih2YXIgcj1lLmxlbmd0aDtyLS07KWlmKFVzKGVbcl1bMF0sdCkpcmV0dXJuIHI7cmV0dXJuLTF9ZnVuY3Rpb24gdGkoZSx0LHIsaSl7cmV0dXJuIHVpKGUsKGZ1bmN0aW9uKGUsbixvKXt0KGksZSxyKGUpLG8pfSkpLGl9ZnVuY3Rpb24gcmkoZSx0KXtyZXR1cm4gZSYma24odCxPYSh0KSxlKX1mdW5jdGlvbiBpaShlLHQscil7Il9fcHJvdG9fXyI9PXQmJmx0P2x0KGUsdCx7Y29uZmlndXJhYmxlOiEwLGVudW1lcmFibGU6ITAsdmFsdWU6cix3cml0YWJsZTohMH0pOmVbdF09cn1mdW5jdGlvbiBuaShlLHQpe2Zvcih2YXIgcj0tMSxvPXQubGVuZ3RoLHM9aShvKSxhPW51bGw9PWU7KytyPG87KXNbcl09YT9uOkFhKGUsdFtyXSk7cmV0dXJuIHN9ZnVuY3Rpb24gb2koZSx0LHIpe3JldHVybiBlPT1lJiYociE9PW4mJihlPWU8PXI/ZTpyKSx0IT09biYmKGU9ZT49dD9lOnQpKSxlfWZ1bmN0aW9uIHNpKGUsdCxyLGksbyxzKXt2YXIgYSxjPTEmdCxsPTImdCx1PTQmdDtpZihyJiYoYT1vP3IoZSxpLG8scyk6cihlKSksYSE9PW4pcmV0dXJuIGE7aWYoIXRhKGUpKXJldHVybiBlO3ZhciBoPUtzKGUpO2lmKGgpe2lmKGE9ZnVuY3Rpb24oZSl7dmFyIHQ9ZS5sZW5ndGgscj1uZXcgZS5jb25zdHJ1Y3Rvcih0KTtyZXR1cm4gdCYmInN0cmluZyI9PXR5cGVvZiBlWzBdJiZCZS5jYWxsKGUsImluZGV4IikmJihyLmluZGV4PWUuaW5kZXgsci5pbnB1dD1lLmlucHV0KSxyfShlKSwhYylyZXR1cm4gQW4oZSxhKX1lbHNle3ZhciBmPWZvKGUpLF89Zj09Ynx8Zj09UztpZihYcyhlKSlyZXR1cm4gU24oZSxjKTtpZihmPT1MfHxmPT1wfHxfJiYhbyl7aWYoYT1sfHxfP3t9OnBvKGUpLCFjKXJldHVybiBsP2Z1bmN0aW9uKGUsdCl7cmV0dXJuIGtuKGUsaG8oZSksdCl9KGUsZnVuY3Rpb24oZSx0KXtyZXR1cm4gZSYma24odCxCYSh0KSxlKX0oYSxlKSk6ZnVuY3Rpb24oZSx0KXtyZXR1cm4ga24oZSx1byhlKSx0KX0oZSxyaShhLGUpKX1lbHNle2lmKCFRZVtmXSlyZXR1cm4gbz9lOnt9O2E9ZnVuY3Rpb24oZSx0LHIpe3ZhciBpLG49ZS5jb25zdHJ1Y3Rvcjtzd2l0Y2godCl7Y2FzZSBUOnJldHVybiBDbihlKTtjYXNlIGc6Y2FzZSB5OnJldHVybiBuZXcgbigrZSk7Y2FzZSBPOnJldHVybiBmdW5jdGlvbihlLHQpe3ZhciByPXQ/Q24oZS5idWZmZXIpOmUuYnVmZmVyO3JldHVybiBuZXcgZS5jb25zdHJ1Y3RvcihyLGUuYnl0ZU9mZnNldCxlLmJ5dGVMZW5ndGgpfShlLHIpO2Nhc2UgQjpjYXNlIEQ6Y2FzZSBQOmNhc2UgSTpjYXNlIEg6Y2FzZSBqOmNhc2UgRjpjYXNlIFc6Y2FzZSBVOnJldHVybiB3bihlLHIpO2Nhc2UgQzpyZXR1cm4gbmV3IG47Y2FzZSB3OmNhc2UgazpyZXR1cm4gbmV3IG4oZSk7Y2FzZSB4OnJldHVybiBmdW5jdGlvbihlKXt2YXIgdD1uZXcgZS5jb25zdHJ1Y3RvcihlLnNvdXJjZSxmZS5leGVjKGUpKTtyZXR1cm4gdC5sYXN0SW5kZXg9ZS5sYXN0SW5kZXgsdH0oZSk7Y2FzZSBBOnJldHVybiBuZXcgbjtjYXNlIE06cmV0dXJuIGk9ZSxJcj9MZShJci5jYWxsKGkpKTp7fX19KGUsZixjKX19c3x8KHM9bmV3IEdyKTt2YXIgZD1zLmdldChlKTtpZihkKXJldHVybiBkO3Muc2V0KGUsYSksYWEoZSk/ZS5mb3JFYWNoKChmdW5jdGlvbihpKXthLmFkZChzaShpLHQscixpLGUscykpfSkpOmlhKGUpJiZlLmZvckVhY2goKGZ1bmN0aW9uKGksbil7YS5zZXQobixzaShpLHQscixuLGUscykpfSkpO3ZhciB2PWg/bjoodT9sP3JvOnRvOmw/QmE6T2EpKGUpO3JldHVybiBtdCh2fHxlLChmdW5jdGlvbihpLG4pe3YmJihpPWVbbj1pXSksUXIoYSxuLHNpKGksdCxyLG4sZSxzKSl9KSksYX1mdW5jdGlvbiBhaShlLHQscil7dmFyIGk9ci5sZW5ndGg7aWYobnVsbD09ZSlyZXR1cm4haTtmb3IoZT1MZShlKTtpLS07KXt2YXIgbz1yW2ldLHM9dFtvXSxhPWVbb107aWYoYT09PW4mJiEobyBpbiBlKXx8IXMoYSkpcmV0dXJuITF9cmV0dXJuITB9ZnVuY3Rpb24gY2koZSx0LHIpe2lmKCJmdW5jdGlvbiIhPXR5cGVvZiBlKXRocm93IG5ldyBBZShvKTtyZXR1cm4gUm8oKGZ1bmN0aW9uKCl7ZS5hcHBseShuLHIpfSksdCl9ZnVuY3Rpb24gbGkoZSx0LHIsaSl7dmFyIG49LTEsbz13dCxzPSEwLGE9ZS5sZW5ndGgsYz1bXSxsPXQubGVuZ3RoO2lmKCFhKXJldHVybiBjO3ImJih0PUV0KHQsTnQocikpKSxpPyhvPUx0LHM9ITEpOnQubGVuZ3RoPj0yMDAmJihvPUt0LHM9ITEsdD1uZXcgVnIodCkpO2U6Zm9yKDsrK248YTspe3ZhciB1PWVbbl0saD1udWxsPT1yP3U6cih1KTtpZih1PWl8fDAhPT11P3U6MCxzJiZoPT1oKXtmb3IodmFyIGY9bDtmLS07KWlmKHRbZl09PT1oKWNvbnRpbnVlIGU7Yy5wdXNoKHUpfWVsc2Ugbyh0LGgsaSl8fGMucHVzaCh1KX1yZXR1cm4gY31qci50ZW1wbGF0ZVNldHRpbmdzPXtlc2NhcGU6WCxldmFsdWF0ZTpaLGludGVycG9sYXRlOkosdmFyaWFibGU6IiIsaW1wb3J0czp7Xzpqcn19LGpyLnByb3RvdHlwZT1Xci5wcm90b3R5cGUsanIucHJvdG90eXBlLmNvbnN0cnVjdG9yPWpyLFVyLnByb3RvdHlwZT1GcihXci5wcm90b3R5cGUpLFVyLnByb3RvdHlwZS5jb25zdHJ1Y3Rvcj1Vcixxci5wcm90b3R5cGU9RnIoV3IucHJvdG90eXBlKSxxci5wcm90b3R5cGUuY29uc3RydWN0b3I9cXIsTnIucHJvdG90eXBlLmNsZWFyPWZ1bmN0aW9uKCl7dGhpcy5fX2RhdGFfXz1Bcj9BcihudWxsKTp7fSx0aGlzLnNpemU9MH0sTnIucHJvdG90eXBlLmRlbGV0ZT1mdW5jdGlvbihlKXt2YXIgdD10aGlzLmhhcyhlKSYmZGVsZXRlIHRoaXMuX19kYXRhX19bZV07cmV0dXJuIHRoaXMuc2l6ZS09dD8xOjAsdH0sTnIucHJvdG90eXBlLmdldD1mdW5jdGlvbihlKXt2YXIgdD10aGlzLl9fZGF0YV9fO2lmKEFyKXt2YXIgcj10W2VdO3JldHVybiByPT09cz9uOnJ9cmV0dXJuIEJlLmNhbGwodCxlKT90W2VdOm59LE5yLnByb3RvdHlwZS5oYXM9ZnVuY3Rpb24oZSl7dmFyIHQ9dGhpcy5fX2RhdGFfXztyZXR1cm4gQXI/dFtlXSE9PW46QmUuY2FsbCh0LGUpfSxOci5wcm90b3R5cGUuc2V0PWZ1bmN0aW9uKGUsdCl7dmFyIHI9dGhpcy5fX2RhdGFfXztyZXR1cm4gdGhpcy5zaXplKz10aGlzLmhhcyhlKT8wOjEscltlXT1BciYmdD09PW4/czp0LHRoaXN9LHpyLnByb3RvdHlwZS5jbGVhcj1mdW5jdGlvbigpe3RoaXMuX19kYXRhX189W10sdGhpcy5zaXplPTB9LHpyLnByb3RvdHlwZS5kZWxldGU9ZnVuY3Rpb24oZSl7dmFyIHQ9dGhpcy5fX2RhdGFfXyxyPWVpKHQsZSk7cmV0dXJuIShyPDB8fChyPT10Lmxlbmd0aC0xP3QucG9wKCk6aXQuY2FsbCh0LHIsMSksLS10aGlzLnNpemUsMCkpfSx6ci5wcm90b3R5cGUuZ2V0PWZ1bmN0aW9uKGUpe3ZhciB0PXRoaXMuX19kYXRhX18scj1laSh0LGUpO3JldHVybiByPDA/bjp0W3JdWzFdfSx6ci5wcm90b3R5cGUuaGFzPWZ1bmN0aW9uKGUpe3JldHVybiBlaSh0aGlzLl9fZGF0YV9fLGUpPi0xfSx6ci5wcm90b3R5cGUuc2V0PWZ1bmN0aW9uKGUsdCl7dmFyIHI9dGhpcy5fX2RhdGFfXyxpPWVpKHIsZSk7cmV0dXJuIGk8MD8oKyt0aGlzLnNpemUsci5wdXNoKFtlLHRdKSk6cltpXVsxXT10LHRoaXN9LEtyLnByb3RvdHlwZS5jbGVhcj1mdW5jdGlvbigpe3RoaXMuc2l6ZT0wLHRoaXMuX19kYXRhX189e2hhc2g6bmV3IE5yLG1hcDpuZXcod3J8fHpyKSxzdHJpbmc6bmV3IE5yfX0sS3IucHJvdG90eXBlLmRlbGV0ZT1mdW5jdGlvbihlKXt2YXIgdD1hbyh0aGlzLGUpLmRlbGV0ZShlKTtyZXR1cm4gdGhpcy5zaXplLT10PzE6MCx0fSxLci5wcm90b3R5cGUuZ2V0PWZ1bmN0aW9uKGUpe3JldHVybiBhbyh0aGlzLGUpLmdldChlKX0sS3IucHJvdG90eXBlLmhhcz1mdW5jdGlvbihlKXtyZXR1cm4gYW8odGhpcyxlKS5oYXMoZSl9LEtyLnByb3RvdHlwZS5zZXQ9ZnVuY3Rpb24oZSx0KXt2YXIgcj1hbyh0aGlzLGUpLGk9ci5zaXplO3JldHVybiByLnNldChlLHQpLHRoaXMuc2l6ZSs9ci5zaXplPT1pPzA6MSx0aGlzfSxWci5wcm90b3R5cGUuYWRkPVZyLnByb3RvdHlwZS5wdXNoPWZ1bmN0aW9uKGUpe3JldHVybiB0aGlzLl9fZGF0YV9fLnNldChlLHMpLHRoaXN9LFZyLnByb3RvdHlwZS5oYXM9ZnVuY3Rpb24oZSl7cmV0dXJuIHRoaXMuX19kYXRhX18uaGFzKGUpfSxHci5wcm90b3R5cGUuY2xlYXI9ZnVuY3Rpb24oKXt0aGlzLl9fZGF0YV9fPW5ldyB6cix0aGlzLnNpemU9MH0sR3IucHJvdG90eXBlLmRlbGV0ZT1mdW5jdGlvbihlKXt2YXIgdD10aGlzLl9fZGF0YV9fLHI9dC5kZWxldGUoZSk7cmV0dXJuIHRoaXMuc2l6ZT10LnNpemUscn0sR3IucHJvdG90eXBlLmdldD1mdW5jdGlvbihlKXtyZXR1cm4gdGhpcy5fX2RhdGFfXy5nZXQoZSl9LEdyLnByb3RvdHlwZS5oYXM9ZnVuY3Rpb24oZSl7cmV0dXJuIHRoaXMuX19kYXRhX18uaGFzKGUpfSxHci5wcm90b3R5cGUuc2V0PWZ1bmN0aW9uKGUsdCl7dmFyIHI9dGhpcy5fX2RhdGFfXztpZihyIGluc3RhbmNlb2YgenIpe3ZhciBpPXIuX19kYXRhX187aWYoIXdyfHxpLmxlbmd0aDwxOTkpcmV0dXJuIGkucHVzaChbZSx0XSksdGhpcy5zaXplPSsrci5zaXplLHRoaXM7cj10aGlzLl9fZGF0YV9fPW5ldyBLcihpKX1yZXR1cm4gci5zZXQoZSx0KSx0aGlzLnNpemU9ci5zaXplLHRoaXN9O3ZhciB1aT1Ubih5aSksaGk9VG4obWksITApO2Z1bmN0aW9uIGZpKGUsdCl7dmFyIHI9ITA7cmV0dXJuIHVpKGUsKGZ1bmN0aW9uKGUsaSxuKXtyZXR1cm4gcj0hIXQoZSxpLG4pfSkpLHJ9ZnVuY3Rpb24gX2koZSx0LHIpe2Zvcih2YXIgaT0tMSxvPWUubGVuZ3RoOysraTxvOyl7dmFyIHM9ZVtpXSxhPXQocyk7aWYobnVsbCE9YSYmKGM9PT1uP2E9PWEmJiFsYShhKTpyKGEsYykpKXZhciBjPWEsbD1zfXJldHVybiBsfWZ1bmN0aW9uIGRpKGUsdCl7dmFyIHI9W107cmV0dXJuIHVpKGUsKGZ1bmN0aW9uKGUsaSxuKXt0KGUsaSxuKSYmci5wdXNoKGUpfSkpLHJ9ZnVuY3Rpb24gcGkoZSx0LHIsaSxuKXt2YXIgbz0tMSxzPWUubGVuZ3RoO2ZvcihyfHwocj12byksbnx8KG49W10pOysrbzxzOyl7dmFyIGE9ZVtvXTt0PjAmJnIoYSk/dD4xP3BpKGEsdC0xLHIsaSxuKTp4dChuLGEpOml8fChuW24ubGVuZ3RoXT1hKX1yZXR1cm4gbn12YXIgdmk9T24oKSxnaT1PbighMCk7ZnVuY3Rpb24geWkoZSx0KXtyZXR1cm4gZSYmdmkoZSx0LE9hKX1mdW5jdGlvbiBtaShlLHQpe3JldHVybiBlJiZnaShlLHQsT2EpfWZ1bmN0aW9uIGJpKGUsdCl7cmV0dXJuIEN0KHQsKGZ1bmN0aW9uKHQpe3JldHVybiAkcyhlW3RdKX0pKX1mdW5jdGlvbiBTaShlLHQpe2Zvcih2YXIgcj0wLGk9KHQ9Z24odCxlKSkubGVuZ3RoO251bGwhPWUmJnI8aTspZT1lW2pvKHRbcisrXSldO3JldHVybiByJiZyPT1pP2U6bn1mdW5jdGlvbiBDaShlLHQscil7dmFyIGk9dChlKTtyZXR1cm4gS3MoZSk/aTp4dChpLHIoZSkpfWZ1bmN0aW9uIHdpKGUpe3JldHVybiBudWxsPT1lP2U9PT1uPyJbb2JqZWN0IFVuZGVmaW5lZF0iOiJbb2JqZWN0IE51bGxdIjphdCYmYXQgaW4gTGUoZSk/ZnVuY3Rpb24oZSl7dmFyIHQ9QmUuY2FsbChlLGF0KSxyPWVbYXRdO3RyeXtlW2F0XT1uO3ZhciBpPSEwfWNhdGNoKGUpe312YXIgbz1JZS5jYWxsKGUpO3JldHVybiBpJiYodD9lW2F0XT1yOmRlbGV0ZSBlW2F0XSksb30oZSk6ZnVuY3Rpb24oZSl7cmV0dXJuIEllLmNhbGwoZSl9KGUpfWZ1bmN0aW9uIExpKGUsdCl7cmV0dXJuIGU+dH1mdW5jdGlvbiBFaShlLHQpe3JldHVybiBudWxsIT1lJiZCZS5jYWxsKGUsdCl9ZnVuY3Rpb24geGkoZSx0KXtyZXR1cm4gbnVsbCE9ZSYmdCBpbiBMZShlKX1mdW5jdGlvbiBBaShlLHQscil7Zm9yKHZhciBvPXI/THQ6d3Qscz1lWzBdLmxlbmd0aCxhPWUubGVuZ3RoLGM9YSxsPWkoYSksdT0xLzAsaD1bXTtjLS07KXt2YXIgZj1lW2NdO2MmJnQmJihmPUV0KGYsTnQodCkpKSx1PWdyKGYubGVuZ3RoLHUpLGxbY109IXImJih0fHxzPj0xMjAmJmYubGVuZ3RoPj0xMjApP25ldyBWcihjJiZmKTpufWY9ZVswXTt2YXIgXz0tMSxkPWxbMF07ZTpmb3IoOysrXzxzJiZoLmxlbmd0aDx1Oyl7dmFyIHA9ZltfXSx2PXQ/dChwKTpwO2lmKHA9cnx8MCE9PXA/cDowLCEoZD9LdChkLHYpOm8oaCx2LHIpKSl7Zm9yKGM9YTstLWM7KXt2YXIgZz1sW2NdO2lmKCEoZz9LdChnLHYpOm8oZVtjXSx2LHIpKSljb250aW51ZSBlfWQmJmQucHVzaCh2KSxoLnB1c2gocCl9fXJldHVybiBofWZ1bmN0aW9uIGtpKGUsdCxyKXt2YXIgaT1udWxsPT0oZT14byhlLHQ9Z24odCxlKSkpP2U6ZVtqbyhKbyh0KSldO3JldHVybiBudWxsPT1pP246Z3QoaSxlLHIpfWZ1bmN0aW9uIE1pKGUpe3JldHVybiByYShlKSYmd2koZSk9PXB9ZnVuY3Rpb24gUmkoZSx0LHIsaSxvKXtyZXR1cm4gZT09PXR8fChudWxsPT1lfHxudWxsPT10fHwhcmEoZSkmJiFyYSh0KT9lIT1lJiZ0IT10OmZ1bmN0aW9uKGUsdCxyLGksbyxzKXt2YXIgYT1LcyhlKSxjPUtzKHQpLGw9YT92OmZvKGUpLHU9Yz92OmZvKHQpLGg9KGw9bD09cD9MOmwpPT1MLGY9KHU9dT09cD9MOnUpPT1MLF89bD09dTtpZihfJiZYcyhlKSl7aWYoIVhzKHQpKXJldHVybiExO2E9ITAsaD0hMX1pZihfJiYhaClyZXR1cm4gc3x8KHM9bmV3IEdyKSxhfHx1YShlKT9RbihlLHQscixpLG8scyk6ZnVuY3Rpb24oZSx0LHIsaSxuLG8scyl7c3dpdGNoKHIpe2Nhc2UgTzppZihlLmJ5dGVMZW5ndGghPXQuYnl0ZUxlbmd0aHx8ZS5ieXRlT2Zmc2V0IT10LmJ5dGVPZmZzZXQpcmV0dXJuITE7ZT1lLmJ1ZmZlcix0PXQuYnVmZmVyO2Nhc2UgVDpyZXR1cm4hKGUuYnl0ZUxlbmd0aCE9dC5ieXRlTGVuZ3RofHwhbyhuZXcgcWUoZSksbmV3IHFlKHQpKSk7Y2FzZSBnOmNhc2UgeTpjYXNlIHc6cmV0dXJuIFVzKCtlLCt0KTtjYXNlIG06cmV0dXJuIGUubmFtZT09dC5uYW1lJiZlLm1lc3NhZ2U9PXQubWVzc2FnZTtjYXNlIHg6Y2FzZSBrOnJldHVybiBlPT10KyIiO2Nhc2UgQzp2YXIgYT1RdDtjYXNlIEE6dmFyIGM9MSZpO2lmKGF8fChhPXJyKSxlLnNpemUhPXQuc2l6ZSYmIWMpcmV0dXJuITE7dmFyIGw9cy5nZXQoZSk7aWYobClyZXR1cm4gbD09dDtpfD0yLHMuc2V0KGUsdCk7dmFyIHU9UW4oYShlKSxhKHQpLGksbixvLHMpO3JldHVybiBzLmRlbGV0ZShlKSx1O2Nhc2UgTTppZihJcilyZXR1cm4gSXIuY2FsbChlKT09SXIuY2FsbCh0KX1yZXR1cm4hMX0oZSx0LGwscixpLG8scyk7aWYoISgxJnIpKXt2YXIgZD1oJiZCZS5jYWxsKGUsIl9fd3JhcHBlZF9fIiksYj1mJiZCZS5jYWxsKHQsIl9fd3JhcHBlZF9fIik7aWYoZHx8Yil7dmFyIFM9ZD9lLnZhbHVlKCk6ZSxFPWI/dC52YWx1ZSgpOnQ7cmV0dXJuIHN8fChzPW5ldyBHciksbyhTLEUscixpLHMpfX1yZXR1cm4hIV8mJihzfHwocz1uZXcgR3IpLGZ1bmN0aW9uKGUsdCxyLGksbyxzKXt2YXIgYT0xJnIsYz10byhlKSxsPWMubGVuZ3RoO2lmKGwhPXRvKHQpLmxlbmd0aCYmIWEpcmV0dXJuITE7Zm9yKHZhciB1PWw7dS0tOyl7dmFyIGg9Y1t1XTtpZighKGE/aCBpbiB0OkJlLmNhbGwodCxoKSkpcmV0dXJuITF9dmFyIGY9cy5nZXQoZSksXz1zLmdldCh0KTtpZihmJiZfKXJldHVybiBmPT10JiZfPT1lO3ZhciBkPSEwO3Muc2V0KGUsdCkscy5zZXQodCxlKTtmb3IodmFyIHA9YTsrK3U8bDspe3ZhciB2PWVbaD1jW3VdXSxnPXRbaF07aWYoaSl2YXIgeT1hP2koZyx2LGgsdCxlLHMpOmkodixnLGgsZSx0LHMpO2lmKCEoeT09PW4/dj09PWd8fG8odixnLHIsaSxzKTp5KSl7ZD0hMTticmVha31wfHwocD0iY29uc3RydWN0b3IiPT1oKX1pZihkJiYhcCl7dmFyIG09ZS5jb25zdHJ1Y3RvcixiPXQuY29uc3RydWN0b3I7bT09Ynx8ISgiY29uc3RydWN0b3IiaW4gZSl8fCEoImNvbnN0cnVjdG9yImluIHQpfHwiZnVuY3Rpb24iPT10eXBlb2YgbSYmbSBpbnN0YW5jZW9mIG0mJiJmdW5jdGlvbiI9PXR5cGVvZiBiJiZiIGluc3RhbmNlb2YgYnx8KGQ9ITEpfXJldHVybiBzLmRlbGV0ZShlKSxzLmRlbGV0ZSh0KSxkfShlLHQscixpLG8scykpfShlLHQscixpLFJpLG8pKX1mdW5jdGlvbiBUaShlLHQscixpKXt2YXIgbz1yLmxlbmd0aCxzPW8sYT0haTtpZihudWxsPT1lKXJldHVybiFzO2ZvcihlPUxlKGUpO28tLTspe3ZhciBjPXJbb107aWYoYSYmY1syXT9jWzFdIT09ZVtjWzBdXTohKGNbMF1pbiBlKSlyZXR1cm4hMX1mb3IoOysrbzxzOyl7dmFyIGw9KGM9cltvXSlbMF0sdT1lW2xdLGg9Y1sxXTtpZihhJiZjWzJdKXtpZih1PT09biYmIShsIGluIGUpKXJldHVybiExfWVsc2V7dmFyIGY9bmV3IEdyO2lmKGkpdmFyIF89aSh1LGgsbCxlLHQsZik7aWYoIShfPT09bj9SaShoLHUsMyxpLGYpOl8pKXJldHVybiExfX1yZXR1cm4hMH1mdW5jdGlvbiBPaShlKXtyZXR1cm4hKCF0YShlKXx8KHQ9ZSxQZSYmUGUgaW4gdCkpJiYoJHMoZSk/RmU6cGUpLnRlc3QoRm8oZSkpO3ZhciB0fWZ1bmN0aW9uIEJpKGUpe3JldHVybiJmdW5jdGlvbiI9PXR5cGVvZiBlP2U6bnVsbD09ZT9uYzoib2JqZWN0Ij09dHlwZW9mIGU/S3MoZSk/amkoZVswXSxlWzFdKTpIaShlKTpfYyhlKX1mdW5jdGlvbiBEaShlKXtpZighQ28oZSkpcmV0dXJuIHByKGUpO3ZhciB0PVtdO2Zvcih2YXIgciBpbiBMZShlKSlCZS5jYWxsKGUscikmJiJjb25zdHJ1Y3RvciIhPXImJnQucHVzaChyKTtyZXR1cm4gdH1mdW5jdGlvbiBQaShlLHQpe3JldHVybiBlPHR9ZnVuY3Rpb24gSWkoZSx0KXt2YXIgcj0tMSxuPUdzKGUpP2koZS5sZW5ndGgpOltdO3JldHVybiB1aShlLChmdW5jdGlvbihlLGksbyl7blsrK3JdPXQoZSxpLG8pfSkpLG59ZnVuY3Rpb24gSGkoZSl7dmFyIHQ9Y28oZSk7cmV0dXJuIDE9PXQubGVuZ3RoJiZ0WzBdWzJdP0xvKHRbMF1bMF0sdFswXVsxXSk6ZnVuY3Rpb24ocil7cmV0dXJuIHI9PT1lfHxUaShyLGUsdCl9fWZ1bmN0aW9uIGppKGUsdCl7cmV0dXJuIG1vKGUpJiZ3byh0KT9MbyhqbyhlKSx0KTpmdW5jdGlvbihyKXt2YXIgaT1BYShyLGUpO3JldHVybiBpPT09biYmaT09PXQ/a2EocixlKTpSaSh0LGksMyl9fWZ1bmN0aW9uIEZpKGUsdCxyLGksbyl7ZSE9PXQmJnZpKHQsKGZ1bmN0aW9uKHMsYSl7aWYob3x8KG89bmV3IEdyKSx0YShzKSkhZnVuY3Rpb24oZSx0LHIsaSxvLHMsYSl7dmFyIGM9a28oZSxyKSxsPWtvKHQsciksdT1hLmdldChsKTtpZih1KSRyKGUscix1KTtlbHNle3ZhciBoPXM/cyhjLGwscisiIixlLHQsYSk6bixmPWg9PT1uO2lmKGYpe3ZhciBfPUtzKGwpLGQ9IV8mJlhzKGwpLHA9IV8mJiFkJiZ1YShsKTtoPWwsX3x8ZHx8cD9LcyhjKT9oPWM6WXMoYyk/aD1BbihjKTpkPyhmPSExLGg9U24obCwhMCkpOnA/KGY9ITEsaD13bihsLCEwKSk6aD1bXTpvYShsKXx8enMobCk/KGg9Yyx6cyhjKT9oPXlhKGMpOnRhKGMpJiYhJHMoYyl8fChoPXBvKGwpKSk6Zj0hMX1mJiYoYS5zZXQobCxoKSxvKGgsbCxpLHMsYSksYS5kZWxldGUobCkpLCRyKGUscixoKX19KGUsdCxhLHIsRmksaSxvKTtlbHNle3ZhciBjPWk/aShrbyhlLGEpLHMsYSsiIixlLHQsbyk6bjtjPT09biYmKGM9cyksJHIoZSxhLGMpfX0pLEJhKX1mdW5jdGlvbiBXaShlLHQpe3ZhciByPWUubGVuZ3RoO2lmKHIpcmV0dXJuIGdvKHQrPXQ8MD9yOjAscik/ZVt0XTpufWZ1bmN0aW9uIFVpKGUsdCxyKXt0PXQubGVuZ3RoP0V0KHQsKGZ1bmN0aW9uKGUpe3JldHVybiBLcyhlKT9mdW5jdGlvbih0KXtyZXR1cm4gU2kodCwxPT09ZS5sZW5ndGg/ZVswXTplKX06ZX0pKTpbbmNdO3ZhciBpPS0xO3Q9RXQodCxOdChzbygpKSk7dmFyIG49SWkoZSwoZnVuY3Rpb24oZSxyLG4pe3ZhciBvPUV0KHQsKGZ1bmN0aW9uKHQpe3JldHVybiB0KGUpfSkpO3JldHVybntjcml0ZXJpYTpvLGluZGV4OisraSx2YWx1ZTplfX0pKTtyZXR1cm4gZnVuY3Rpb24oZSx0KXt2YXIgaT1lLmxlbmd0aDtmb3IoZS5zb3J0KChmdW5jdGlvbihlLHQpe3JldHVybiBmdW5jdGlvbihlLHQscil7Zm9yKHZhciBpPS0xLG49ZS5jcml0ZXJpYSxvPXQuY3JpdGVyaWEscz1uLmxlbmd0aCxhPXIubGVuZ3RoOysraTxzOyl7dmFyIGM9TG4obltpXSxvW2ldKTtpZihjKXJldHVybiBpPj1hP2M6YyooImRlc2MiPT1yW2ldPy0xOjEpfXJldHVybiBlLmluZGV4LXQuaW5kZXh9KGUsdCxyKX0pKTtpLS07KWVbaV09ZVtpXS52YWx1ZTtyZXR1cm4gZX0obil9ZnVuY3Rpb24gcWkoZSx0LHIpe2Zvcih2YXIgaT0tMSxuPXQubGVuZ3RoLG89e307KytpPG47KXt2YXIgcz10W2ldLGE9U2koZSxzKTtyKGEscykmJlppKG8sZ24ocyxlKSxhKX1yZXR1cm4gb31mdW5jdGlvbiBOaShlLHQscixpKXt2YXIgbj1pP0R0OkJ0LG89LTEscz10Lmxlbmd0aCxhPWU7Zm9yKGU9PT10JiYodD1Bbih0KSksciYmKGE9RXQoZSxOdChyKSkpOysrbzxzOylmb3IodmFyIGM9MCxsPXRbb10sdT1yP3IobCk6bDsoYz1uKGEsdSxjLGkpKT4tMTspYSE9PWUmJml0LmNhbGwoYSxjLDEpLGl0LmNhbGwoZSxjLDEpO3JldHVybiBlfWZ1bmN0aW9uIHppKGUsdCl7Zm9yKHZhciByPWU/dC5sZW5ndGg6MCxpPXItMTtyLS07KXt2YXIgbj10W3JdO2lmKHI9PWl8fG4hPT1vKXt2YXIgbz1uO2dvKG4pP2l0LmNhbGwoZSxuLDEpOmxuKGUsbil9fXJldHVybiBlfWZ1bmN0aW9uIEtpKGUsdCl7cmV0dXJuIGUrdXIoYnIoKSoodC1lKzEpKX1mdW5jdGlvbiBWaShlLHQpe3ZhciByPSIiO2lmKCFlfHx0PDF8fHQ+aClyZXR1cm4gcjtkb3t0JTImJihyKz1lKSwodD11cih0LzIpKSYmKGUrPWUpfXdoaWxlKHQpO3JldHVybiByfWZ1bmN0aW9uIEdpKGUsdCl7cmV0dXJuIFRvKEVvKGUsdCxuYyksZSsiIil9ZnVuY3Rpb24gWWkoZSl7cmV0dXJuIFhyKFVhKGUpKX1mdW5jdGlvbiBYaShlLHQpe3ZhciByPVVhKGUpO3JldHVybiBEbyhyLG9pKHQsMCxyLmxlbmd0aCkpfWZ1bmN0aW9uIFppKGUsdCxyLGkpe2lmKCF0YShlKSlyZXR1cm4gZTtmb3IodmFyIG89LTEscz0odD1nbih0LGUpKS5sZW5ndGgsYT1zLTEsYz1lO251bGwhPWMmJisrbzxzOyl7dmFyIGw9am8odFtvXSksdT1yO2lmKCJfX3Byb3RvX18iPT09bHx8ImNvbnN0cnVjdG9yIj09PWx8fCJwcm90b3R5cGUiPT09bClyZXR1cm4gZTtpZihvIT1hKXt2YXIgaD1jW2xdOyh1PWk/aShoLGwsYyk6bik9PT1uJiYodT10YShoKT9oOmdvKHRbbysxXSk/W106e30pfVFyKGMsbCx1KSxjPWNbbF19cmV0dXJuIGV9dmFyIEppPWtyP2Z1bmN0aW9uKGUsdCl7cmV0dXJuIGtyLnNldChlLHQpLGV9Om5jLCRpPWx0P2Z1bmN0aW9uKGUsdCl7cmV0dXJuIGx0KGUsInRvU3RyaW5nIix7Y29uZmlndXJhYmxlOiEwLGVudW1lcmFibGU6ITEsdmFsdWU6dGModCksd3JpdGFibGU6ITB9KX06bmM7ZnVuY3Rpb24gUWkoZSl7cmV0dXJuIERvKFVhKGUpKX1mdW5jdGlvbiBlbihlLHQscil7dmFyIG49LTEsbz1lLmxlbmd0aDt0PDAmJih0PS10Pm8/MDpvK3QpLChyPXI+bz9vOnIpPDAmJihyKz1vKSxvPXQ+cj8wOnItdD4+PjAsdD4+Pj0wO2Zvcih2YXIgcz1pKG8pOysrbjxvOylzW25dPWVbbit0XTtyZXR1cm4gc31mdW5jdGlvbiB0bihlLHQpe3ZhciByO3JldHVybiB1aShlLChmdW5jdGlvbihlLGksbil7cmV0dXJuIShyPXQoZSxpLG4pKX0pKSwhIXJ9ZnVuY3Rpb24gcm4oZSx0LHIpe3ZhciBpPTAsbj1udWxsPT1lP2k6ZS5sZW5ndGg7aWYoIm51bWJlciI9PXR5cGVvZiB0JiZ0PT10JiZuPD0yMTQ3NDgzNjQ3KXtmb3IoO2k8bjspe3ZhciBvPWkrbj4+PjEscz1lW29dO251bGwhPT1zJiYhbGEocykmJihyP3M8PXQ6czx0KT9pPW8rMTpuPW99cmV0dXJuIG59cmV0dXJuIG5uKGUsdCxuYyxyKX1mdW5jdGlvbiBubihlLHQscixpKXt2YXIgbz0wLHM9bnVsbD09ZT8wOmUubGVuZ3RoO2lmKDA9PT1zKXJldHVybiAwO2Zvcih2YXIgYT0odD1yKHQpKSE9dCxjPW51bGw9PT10LGw9bGEodCksdT10PT09bjtvPHM7KXt2YXIgaD11cigobytzKS8yKSxmPXIoZVtoXSksXz1mIT09bixkPW51bGw9PT1mLHA9Zj09Zix2PWxhKGYpO2lmKGEpdmFyIGc9aXx8cDtlbHNlIGc9dT9wJiYoaXx8Xyk6Yz9wJiZfJiYoaXx8IWQpOmw/cCYmXyYmIWQmJihpfHwhdik6IWQmJiF2JiYoaT9mPD10OmY8dCk7Zz9vPWgrMTpzPWh9cmV0dXJuIGdyKHMsNDI5NDk2NzI5NCl9ZnVuY3Rpb24gb24oZSx0KXtmb3IodmFyIHI9LTEsaT1lLmxlbmd0aCxuPTAsbz1bXTsrK3I8aTspe3ZhciBzPWVbcl0sYT10P3Qocyk6cztpZighcnx8IVVzKGEsYykpe3ZhciBjPWE7b1tuKytdPTA9PT1zPzA6c319cmV0dXJuIG99ZnVuY3Rpb24gc24oZSl7cmV0dXJuIm51bWJlciI9PXR5cGVvZiBlP2U6bGEoZSk/ZjorZX1mdW5jdGlvbiBhbihlKXtpZigic3RyaW5nIj09dHlwZW9mIGUpcmV0dXJuIGU7aWYoS3MoZSkpcmV0dXJuIEV0KGUsYW4pKyIiO2lmKGxhKGUpKXJldHVybiBIcj9Ici5jYWxsKGUpOiIiO3ZhciB0PWUrIiI7cmV0dXJuIjAiPT10JiYxL2U9PS0xLzA/Ii0wIjp0fWZ1bmN0aW9uIGNuKGUsdCxyKXt2YXIgaT0tMSxuPXd0LG89ZS5sZW5ndGgscz0hMCxhPVtdLGM9YTtpZihyKXM9ITEsbj1MdDtlbHNlIGlmKG8+PTIwMCl7dmFyIGw9dD9udWxsOkduKGUpO2lmKGwpcmV0dXJuIHJyKGwpO3M9ITEsbj1LdCxjPW5ldyBWcn1lbHNlIGM9dD9bXTphO2U6Zm9yKDsrK2k8bzspe3ZhciB1PWVbaV0saD10P3QodSk6dTtpZih1PXJ8fDAhPT11P3U6MCxzJiZoPT1oKXtmb3IodmFyIGY9Yy5sZW5ndGg7Zi0tOylpZihjW2ZdPT09aCljb250aW51ZSBlO3QmJmMucHVzaChoKSxhLnB1c2godSl9ZWxzZSBuKGMsaCxyKXx8KGMhPT1hJiZjLnB1c2goaCksYS5wdXNoKHUpKX1yZXR1cm4gYX1mdW5jdGlvbiBsbihlLHQpe3JldHVybiBudWxsPT0oZT14byhlLHQ9Z24odCxlKSkpfHxkZWxldGUgZVtqbyhKbyh0KSldfWZ1bmN0aW9uIHVuKGUsdCxyLGkpe3JldHVybiBaaShlLHQscihTaShlLHQpKSxpKX1mdW5jdGlvbiBobihlLHQscixpKXtmb3IodmFyIG49ZS5sZW5ndGgsbz1pP246LTE7KGk/by0tOisrbzxuKSYmdChlW29dLG8sZSk7KTtyZXR1cm4gcj9lbihlLGk/MDpvLGk/bysxOm4pOmVuKGUsaT9vKzE6MCxpP246byl9ZnVuY3Rpb24gZm4oZSx0KXt2YXIgcj1lO3JldHVybiByIGluc3RhbmNlb2YgcXImJihyPXIudmFsdWUoKSksQXQodCwoZnVuY3Rpb24oZSx0KXtyZXR1cm4gdC5mdW5jLmFwcGx5KHQudGhpc0FyZyx4dChbZV0sdC5hcmdzKSl9KSxyKX1mdW5jdGlvbiBfbihlLHQscil7dmFyIG49ZS5sZW5ndGg7aWYobjwyKXJldHVybiBuP2NuKGVbMF0pOltdO2Zvcih2YXIgbz0tMSxzPWkobik7KytvPG47KWZvcih2YXIgYT1lW29dLGM9LTE7KytjPG47KWMhPW8mJihzW29dPWxpKHNbb118fGEsZVtjXSx0LHIpKTtyZXR1cm4gY24ocGkocywxKSx0LHIpfWZ1bmN0aW9uIGRuKGUsdCxyKXtmb3IodmFyIGk9LTEsbz1lLmxlbmd0aCxzPXQubGVuZ3RoLGE9e307KytpPG87KXt2YXIgYz1pPHM/dFtpXTpuO3IoYSxlW2ldLGMpfXJldHVybiBhfWZ1bmN0aW9uIHBuKGUpe3JldHVybiBZcyhlKT9lOltdfWZ1bmN0aW9uIHZuKGUpe3JldHVybiJmdW5jdGlvbiI9PXR5cGVvZiBlP2U6bmN9ZnVuY3Rpb24gZ24oZSx0KXtyZXR1cm4gS3MoZSk/ZTptbyhlLHQpP1tlXTpIbyhtYShlKSl9dmFyIHluPUdpO2Z1bmN0aW9uIG1uKGUsdCxyKXt2YXIgaT1lLmxlbmd0aDtyZXR1cm4gcj1yPT09bj9pOnIsIXQmJnI+PWk/ZTplbihlLHQscil9dmFyIGJuPXV0fHxmdW5jdGlvbihlKXtyZXR1cm4gb3QuY2xlYXJUaW1lb3V0KGUpfTtmdW5jdGlvbiBTbihlLHQpe2lmKHQpcmV0dXJuIGUuc2xpY2UoKTt2YXIgcj1lLmxlbmd0aCxpPU5lP05lKHIpOm5ldyBlLmNvbnN0cnVjdG9yKHIpO3JldHVybiBlLmNvcHkoaSksaX1mdW5jdGlvbiBDbihlKXt2YXIgdD1uZXcgZS5jb25zdHJ1Y3RvcihlLmJ5dGVMZW5ndGgpO3JldHVybiBuZXcgcWUodCkuc2V0KG5ldyBxZShlKSksdH1mdW5jdGlvbiB3bihlLHQpe3ZhciByPXQ/Q24oZS5idWZmZXIpOmUuYnVmZmVyO3JldHVybiBuZXcgZS5jb25zdHJ1Y3RvcihyLGUuYnl0ZU9mZnNldCxlLmxlbmd0aCl9ZnVuY3Rpb24gTG4oZSx0KXtpZihlIT09dCl7dmFyIHI9ZSE9PW4saT1udWxsPT09ZSxvPWU9PWUscz1sYShlKSxhPXQhPT1uLGM9bnVsbD09PXQsbD10PT10LHU9bGEodCk7aWYoIWMmJiF1JiYhcyYmZT50fHxzJiZhJiZsJiYhYyYmIXV8fGkmJmEmJmx8fCFyJiZsfHwhbylyZXR1cm4gMTtpZighaSYmIXMmJiF1JiZlPHR8fHUmJnImJm8mJiFpJiYhc3x8YyYmciYmb3x8IWEmJm98fCFsKXJldHVybi0xfXJldHVybiAwfWZ1bmN0aW9uIEVuKGUsdCxyLG4pe2Zvcih2YXIgbz0tMSxzPWUubGVuZ3RoLGE9ci5sZW5ndGgsYz0tMSxsPXQubGVuZ3RoLHU9dnIocy1hLDApLGg9aShsK3UpLGY9IW47KytjPGw7KWhbY109dFtjXTtmb3IoOysrbzxhOykoZnx8bzxzKSYmKGhbcltvXV09ZVtvXSk7Zm9yKDt1LS07KWhbYysrXT1lW28rK107cmV0dXJuIGh9ZnVuY3Rpb24geG4oZSx0LHIsbil7Zm9yKHZhciBvPS0xLHM9ZS5sZW5ndGgsYT0tMSxjPXIubGVuZ3RoLGw9LTEsdT10Lmxlbmd0aCxoPXZyKHMtYywwKSxmPWkoaCt1KSxfPSFuOysrbzxoOylmW29dPWVbb107Zm9yKHZhciBkPW87KytsPHU7KWZbZCtsXT10W2xdO2Zvcig7KythPGM7KShffHxvPHMpJiYoZltkK3JbYV1dPWVbbysrXSk7cmV0dXJuIGZ9ZnVuY3Rpb24gQW4oZSx0KXt2YXIgcj0tMSxuPWUubGVuZ3RoO2Zvcih0fHwodD1pKG4pKTsrK3I8bjspdFtyXT1lW3JdO3JldHVybiB0fWZ1bmN0aW9uIGtuKGUsdCxyLGkpe3ZhciBvPSFyO3J8fChyPXt9KTtmb3IodmFyIHM9LTEsYT10Lmxlbmd0aDsrK3M8YTspe3ZhciBjPXRbc10sbD1pP2kocltjXSxlW2NdLGMscixlKTpuO2w9PT1uJiYobD1lW2NdKSxvP2lpKHIsYyxsKTpRcihyLGMsbCl9cmV0dXJuIHJ9ZnVuY3Rpb24gTW4oZSx0KXtyZXR1cm4gZnVuY3Rpb24ocixpKXt2YXIgbj1LcyhyKT95dDp0aSxvPXQ/dCgpOnt9O3JldHVybiBuKHIsZSxzbyhpLDIpLG8pfX1mdW5jdGlvbiBSbihlKXtyZXR1cm4gR2koKGZ1bmN0aW9uKHQscil7dmFyIGk9LTEsbz1yLmxlbmd0aCxzPW8+MT9yW28tMV06bixhPW8+Mj9yWzJdOm47Zm9yKHM9ZS5sZW5ndGg+MyYmImZ1bmN0aW9uIj09dHlwZW9mIHM/KG8tLSxzKTpuLGEmJnlvKHJbMF0sclsxXSxhKSYmKHM9bzwzP246cyxvPTEpLHQ9TGUodCk7KytpPG87KXt2YXIgYz1yW2ldO2MmJmUodCxjLGkscyl9cmV0dXJuIHR9KSl9ZnVuY3Rpb24gVG4oZSx0KXtyZXR1cm4gZnVuY3Rpb24ocixpKXtpZihudWxsPT1yKXJldHVybiByO2lmKCFHcyhyKSlyZXR1cm4gZShyLGkpO2Zvcih2YXIgbj1yLmxlbmd0aCxvPXQ/bjotMSxzPUxlKHIpOyh0P28tLTorK288bikmJiExIT09aShzW29dLG8scyk7KTtyZXR1cm4gcn19ZnVuY3Rpb24gT24oZSl7cmV0dXJuIGZ1bmN0aW9uKHQscixpKXtmb3IodmFyIG49LTEsbz1MZSh0KSxzPWkodCksYT1zLmxlbmd0aDthLS07KXt2YXIgYz1zW2U/YTorK25dO2lmKCExPT09cihvW2NdLGMsbykpYnJlYWt9cmV0dXJuIHR9fWZ1bmN0aW9uIEJuKGUpe3JldHVybiBmdW5jdGlvbih0KXt2YXIgcj0kdCh0PW1hKHQpKT9vcih0KTpuLGk9cj9yWzBdOnQuY2hhckF0KDApLG89cj9tbihyLDEpLmpvaW4oIiIpOnQuc2xpY2UoMSk7cmV0dXJuIGlbZV0oKStvfX1mdW5jdGlvbiBEbihlKXtyZXR1cm4gZnVuY3Rpb24odCl7cmV0dXJuIEF0KCRhKHphKHQpLnJlcGxhY2UoemUsIiIpKSxlLCIiKX19ZnVuY3Rpb24gUG4oZSl7cmV0dXJuIGZ1bmN0aW9uKCl7dmFyIHQ9YXJndW1lbnRzO3N3aXRjaCh0Lmxlbmd0aCl7Y2FzZSAwOnJldHVybiBuZXcgZTtjYXNlIDE6cmV0dXJuIG5ldyBlKHRbMF0pO2Nhc2UgMjpyZXR1cm4gbmV3IGUodFswXSx0WzFdKTtjYXNlIDM6cmV0dXJuIG5ldyBlKHRbMF0sdFsxXSx0WzJdKTtjYXNlIDQ6cmV0dXJuIG5ldyBlKHRbMF0sdFsxXSx0WzJdLHRbM10pO2Nhc2UgNTpyZXR1cm4gbmV3IGUodFswXSx0WzFdLHRbMl0sdFszXSx0WzRdKTtjYXNlIDY6cmV0dXJuIG5ldyBlKHRbMF0sdFsxXSx0WzJdLHRbM10sdFs0XSx0WzVdKTtjYXNlIDc6cmV0dXJuIG5ldyBlKHRbMF0sdFsxXSx0WzJdLHRbM10sdFs0XSx0WzVdLHRbNl0pfXZhciByPUZyKGUucHJvdG90eXBlKSxpPWUuYXBwbHkocix0KTtyZXR1cm4gdGEoaSk/aTpyfX1mdW5jdGlvbiBJbihlKXtyZXR1cm4gZnVuY3Rpb24odCxyLGkpe3ZhciBvPUxlKHQpO2lmKCFHcyh0KSl7dmFyIHM9c28ociwzKTt0PU9hKHQpLHI9ZnVuY3Rpb24oZSl7cmV0dXJuIHMob1tlXSxlLG8pfX12YXIgYT1lKHQscixpKTtyZXR1cm4gYT4tMT9vW3M/dFthXTphXTpufX1mdW5jdGlvbiBIbihlKXtyZXR1cm4gZW8oKGZ1bmN0aW9uKHQpe3ZhciByPXQubGVuZ3RoLGk9cixzPVVyLnByb3RvdHlwZS50aHJ1O2ZvcihlJiZ0LnJldmVyc2UoKTtpLS07KXt2YXIgYT10W2ldO2lmKCJmdW5jdGlvbiIhPXR5cGVvZiBhKXRocm93IG5ldyBBZShvKTtpZihzJiYhYyYmIndyYXBwZXIiPT1ubyhhKSl2YXIgYz1uZXcgVXIoW10sITApfWZvcihpPWM/aTpyOysraTxyOyl7dmFyIGw9bm8oYT10W2ldKSx1PSJ3cmFwcGVyIj09bD9pbyhhKTpuO2M9dSYmYm8odVswXSkmJjQyND09dVsxXSYmIXVbNF0ubGVuZ3RoJiYxPT11WzldP2Nbbm8odVswXSldLmFwcGx5KGMsdVszXSk6MT09YS5sZW5ndGgmJmJvKGEpP2NbbF0oKTpjLnRocnUoYSl9cmV0dXJuIGZ1bmN0aW9uKCl7dmFyIGU9YXJndW1lbnRzLGk9ZVswXTtpZihjJiYxPT1lLmxlbmd0aCYmS3MoaSkpcmV0dXJuIGMucGxhbnQoaSkudmFsdWUoKTtmb3IodmFyIG49MCxvPXI/dFtuXS5hcHBseSh0aGlzLGUpOmk7KytuPHI7KW89dFtuXS5jYWxsKHRoaXMsbyk7cmV0dXJuIG99fSkpfWZ1bmN0aW9uIGpuKGUsdCxyLG8scyxhLGMsdSxoLGYpe3ZhciBfPXQmbCxkPTEmdCxwPTImdCx2PTI0JnQsZz01MTImdCx5PXA/bjpQbihlKTtyZXR1cm4gZnVuY3Rpb24gbigpe2Zvcih2YXIgbD1hcmd1bWVudHMubGVuZ3RoLG09aShsKSxiPWw7Yi0tOyltW2JdPWFyZ3VtZW50c1tiXTtpZih2KXZhciBTPW9vKG4pLEM9WXQobSxTKTtpZihvJiYobT1FbihtLG8scyx2KSksYSYmKG09eG4obSxhLGMsdikpLGwtPUMsdiYmbDxmKXt2YXIgdz10cihtLFMpO3JldHVybiBLbihlLHQsam4sbi5wbGFjZWhvbGRlcixyLG0sdyx1LGgsZi1sKX12YXIgTD1kP3I6dGhpcyxFPXA/TFtlXTplO3JldHVybiBsPW0ubGVuZ3RoLHU/bT1BbyhtLHUpOmcmJmw+MSYmbS5yZXZlcnNlKCksXyYmaDxsJiYobS5sZW5ndGg9aCksdGhpcyYmdGhpcyE9PW90JiZ0aGlzIGluc3RhbmNlb2YgbiYmKEU9eXx8UG4oRSkpLEUuYXBwbHkoTCxtKX19ZnVuY3Rpb24gRm4oZSx0KXtyZXR1cm4gZnVuY3Rpb24ocixpKXtyZXR1cm4gZnVuY3Rpb24oZSx0LHIsaSl7cmV0dXJuIHlpKGUsKGZ1bmN0aW9uKGUsbixvKXt0KGkscihlKSxuLG8pfSkpLGl9KHIsZSx0KGkpLHt9KX19ZnVuY3Rpb24gV24oZSx0KXtyZXR1cm4gZnVuY3Rpb24ocixpKXt2YXIgbztpZihyPT09biYmaT09PW4pcmV0dXJuIHQ7aWYociE9PW4mJihvPXIpLGkhPT1uKXtpZihvPT09bilyZXR1cm4gaTsic3RyaW5nIj09dHlwZW9mIHJ8fCJzdHJpbmciPT10eXBlb2YgaT8ocj1hbihyKSxpPWFuKGkpKToocj1zbihyKSxpPXNuKGkpKSxvPWUocixpKX1yZXR1cm4gb319ZnVuY3Rpb24gVW4oZSl7cmV0dXJuIGVvKChmdW5jdGlvbih0KXtyZXR1cm4gdD1FdCh0LE50KHNvKCkpKSxHaSgoZnVuY3Rpb24ocil7dmFyIGk9dGhpcztyZXR1cm4gZSh0LChmdW5jdGlvbihlKXtyZXR1cm4gZ3QoZSxpLHIpfSkpfSkpfSkpfWZ1bmN0aW9uIHFuKGUsdCl7dmFyIHI9KHQ9dD09PW4/IiAiOmFuKHQpKS5sZW5ndGg7aWYocjwyKXJldHVybiByP1ZpKHQsZSk6dDt2YXIgaT1WaSh0LGxyKGUvbnIodCkpKTtyZXR1cm4gJHQodCk/bW4ob3IoaSksMCxlKS5qb2luKCIiKTppLnNsaWNlKDAsZSl9ZnVuY3Rpb24gTm4oZSl7cmV0dXJuIGZ1bmN0aW9uKHQscixvKXtyZXR1cm4gbyYmIm51bWJlciIhPXR5cGVvZiBvJiZ5byh0LHIsbykmJihyPW89biksdD1kYSh0KSxyPT09bj8ocj10LHQ9MCk6cj1kYShyKSxmdW5jdGlvbihlLHQscixuKXtmb3IodmFyIG89LTEscz12cihscigodC1lKS8ocnx8MSkpLDApLGE9aShzKTtzLS07KWFbbj9zOisrb109ZSxlKz1yO3JldHVybiBhfSh0LHIsbz1vPT09bj90PHI/MTotMTpkYShvKSxlKX19ZnVuY3Rpb24gem4oZSl7cmV0dXJuIGZ1bmN0aW9uKHQscil7cmV0dXJuInN0cmluZyI9PXR5cGVvZiB0JiYic3RyaW5nIj09dHlwZW9mIHJ8fCh0PWdhKHQpLHI9Z2EocikpLGUodCxyKX19ZnVuY3Rpb24gS24oZSx0LHIsaSxvLHMsYSxsLHUsaCl7dmFyIGY9OCZ0O3R8PWY/Yzo2NCw0Jih0Jj1+KGY/NjQ6YykpfHwodCY9LTQpO3ZhciBfPVtlLHQsbyxmP3M6bixmP2E6bixmP246cyxmP246YSxsLHUsaF0sZD1yLmFwcGx5KG4sXyk7cmV0dXJuIGJvKGUpJiZNbyhkLF8pLGQucGxhY2Vob2xkZXI9aSxPbyhkLGUsdCl9ZnVuY3Rpb24gVm4oZSl7dmFyIHQ9d2VbZV07cmV0dXJuIGZ1bmN0aW9uKGUscil7aWYoZT1nYShlKSwocj1udWxsPT1yPzA6Z3IocGEociksMjkyKSkmJl9yKGUpKXt2YXIgaT0obWEoZSkrImUiKS5zcGxpdCgiZSIpO3JldHVybisoKGk9KG1hKHQoaVswXSsiZSIrKCtpWzFdK3IpKSkrImUiKS5zcGxpdCgiZSIpKVswXSsiZSIrKCtpWzFdLXIpKX1yZXR1cm4gdChlKX19dmFyIEduPUVyJiYxL3JyKG5ldyBFcihbLC0wXSkpWzFdPT11P2Z1bmN0aW9uKGUpe3JldHVybiBuZXcgRXIoZSl9OmxjO2Z1bmN0aW9uIFluKGUpe3JldHVybiBmdW5jdGlvbih0KXt2YXIgcj1mbyh0KTtyZXR1cm4gcj09Qz9RdCh0KTpyPT1BP2lyKHQpOmZ1bmN0aW9uKGUsdCl7cmV0dXJuIEV0KHQsKGZ1bmN0aW9uKHQpe3JldHVyblt0LGVbdF1dfSkpfSh0LGUodCkpfX1mdW5jdGlvbiBYbihlLHQscixzLHUsaCxmLF8pe3ZhciBkPTImdDtpZighZCYmImZ1bmN0aW9uIiE9dHlwZW9mIGUpdGhyb3cgbmV3IEFlKG8pO3ZhciBwPXM/cy5sZW5ndGg6MDtpZihwfHwodCY9LTk3LHM9dT1uKSxmPWY9PT1uP2Y6dnIocGEoZiksMCksXz1fPT09bj9fOnBhKF8pLHAtPXU/dS5sZW5ndGg6MCw2NCZ0KXt2YXIgdj1zLGc9dTtzPXU9bn12YXIgeT1kP246aW8oZSksbT1bZSx0LHIscyx1LHYsZyxoLGYsX107aWYoeSYmZnVuY3Rpb24oZSx0KXt2YXIgcj1lWzFdLGk9dFsxXSxuPXJ8aSxvPW48MTMxLHM9aT09bCYmOD09cnx8aT09bCYmMjU2PT1yJiZlWzddLmxlbmd0aDw9dFs4XXx8Mzg0PT1pJiZ0WzddLmxlbmd0aDw9dFs4XSYmOD09cjtpZighbyYmIXMpcmV0dXJuIGU7MSZpJiYoZVsyXT10WzJdLG58PTEmcj8wOjQpO3ZhciBjPXRbM107aWYoYyl7dmFyIHU9ZVszXTtlWzNdPXU/RW4odSxjLHRbNF0pOmMsZVs0XT11P3RyKGVbM10sYSk6dFs0XX0oYz10WzVdKSYmKHU9ZVs1XSxlWzVdPXU/eG4odSxjLHRbNl0pOmMsZVs2XT11P3RyKGVbNV0sYSk6dFs2XSksKGM9dFs3XSkmJihlWzddPWMpLGkmbCYmKGVbOF09bnVsbD09ZVs4XT90WzhdOmdyKGVbOF0sdFs4XSkpLG51bGw9PWVbOV0mJihlWzldPXRbOV0pLGVbMF09dFswXSxlWzFdPW59KG0seSksZT1tWzBdLHQ9bVsxXSxyPW1bMl0scz1tWzNdLHU9bVs0XSwhKF89bVs5XT1tWzldPT09bj9kPzA6ZS5sZW5ndGg6dnIobVs5XS1wLDApKSYmMjQmdCYmKHQmPS0yNSksdCYmMSE9dCliPTg9PXR8fDE2PT10P2Z1bmN0aW9uKGUsdCxyKXt2YXIgbz1QbihlKTtyZXR1cm4gZnVuY3Rpb24gcygpe2Zvcih2YXIgYT1hcmd1bWVudHMubGVuZ3RoLGM9aShhKSxsPWEsdT1vbyhzKTtsLS07KWNbbF09YXJndW1lbnRzW2xdO3ZhciBoPWE8MyYmY1swXSE9PXUmJmNbYS0xXSE9PXU/W106dHIoYyx1KTtyZXR1cm4oYS09aC5sZW5ndGgpPHI/S24oZSx0LGpuLHMucGxhY2Vob2xkZXIsbixjLGgsbixuLHItYSk6Z3QodGhpcyYmdGhpcyE9PW90JiZ0aGlzIGluc3RhbmNlb2Ygcz9vOmUsdGhpcyxjKX19KGUsdCxfKTp0IT1jJiYzMyE9dHx8dS5sZW5ndGg/am4uYXBwbHkobixtKTpmdW5jdGlvbihlLHQscixuKXt2YXIgbz0xJnQscz1QbihlKTtyZXR1cm4gZnVuY3Rpb24gdCgpe2Zvcih2YXIgYT0tMSxjPWFyZ3VtZW50cy5sZW5ndGgsbD0tMSx1PW4ubGVuZ3RoLGg9aSh1K2MpLGY9dGhpcyYmdGhpcyE9PW90JiZ0aGlzIGluc3RhbmNlb2YgdD9zOmU7KytsPHU7KWhbbF09bltsXTtmb3IoO2MtLTspaFtsKytdPWFyZ3VtZW50c1srK2FdO3JldHVybiBndChmLG8/cjp0aGlzLGgpfX0oZSx0LHIscyk7ZWxzZSB2YXIgYj1mdW5jdGlvbihlLHQscil7dmFyIGk9MSZ0LG49UG4oZSk7cmV0dXJuIGZ1bmN0aW9uIHQoKXtyZXR1cm4odGhpcyYmdGhpcyE9PW90JiZ0aGlzIGluc3RhbmNlb2YgdD9uOmUpLmFwcGx5KGk/cjp0aGlzLGFyZ3VtZW50cyl9fShlLHQscik7cmV0dXJuIE9vKCh5P0ppOk1vKShiLG0pLGUsdCl9ZnVuY3Rpb24gWm4oZSx0LHIsaSl7cmV0dXJuIGU9PT1ufHxVcyhlLFJlW3JdKSYmIUJlLmNhbGwoaSxyKT90OmV9ZnVuY3Rpb24gSm4oZSx0LHIsaSxvLHMpe3JldHVybiB0YShlKSYmdGEodCkmJihzLnNldCh0LGUpLEZpKGUsdCxuLEpuLHMpLHMuZGVsZXRlKHQpKSxlfWZ1bmN0aW9uICRuKGUpe3JldHVybiBvYShlKT9uOmV9ZnVuY3Rpb24gUW4oZSx0LHIsaSxvLHMpe3ZhciBhPTEmcixjPWUubGVuZ3RoLGw9dC5sZW5ndGg7aWYoYyE9bCYmIShhJiZsPmMpKXJldHVybiExO3ZhciB1PXMuZ2V0KGUpLGg9cy5nZXQodCk7aWYodSYmaClyZXR1cm4gdT09dCYmaD09ZTt2YXIgZj0tMSxfPSEwLGQ9MiZyP25ldyBWcjpuO2ZvcihzLnNldChlLHQpLHMuc2V0KHQsZSk7KytmPGM7KXt2YXIgcD1lW2ZdLHY9dFtmXTtpZihpKXZhciBnPWE/aSh2LHAsZix0LGUscyk6aShwLHYsZixlLHQscyk7aWYoZyE9PW4pe2lmKGcpY29udGludWU7Xz0hMTticmVha31pZihkKXtpZighTXQodCwoZnVuY3Rpb24oZSx0KXtpZighS3QoZCx0KSYmKHA9PT1lfHxvKHAsZSxyLGkscykpKXJldHVybiBkLnB1c2godCl9KSkpe189ITE7YnJlYWt9fWVsc2UgaWYocCE9PXYmJiFvKHAsdixyLGkscykpe189ITE7YnJlYWt9fXJldHVybiBzLmRlbGV0ZShlKSxzLmRlbGV0ZSh0KSxffWZ1bmN0aW9uIGVvKGUpe3JldHVybiBUbyhFbyhlLG4sVm8pLGUrIiIpfWZ1bmN0aW9uIHRvKGUpe3JldHVybiBDaShlLE9hLHVvKX1mdW5jdGlvbiBybyhlKXtyZXR1cm4gQ2koZSxCYSxobyl9dmFyIGlvPWtyP2Z1bmN0aW9uKGUpe3JldHVybiBrci5nZXQoZSl9OmxjO2Z1bmN0aW9uIG5vKGUpe2Zvcih2YXIgdD1lLm5hbWUrIiIscj1Nclt0XSxpPUJlLmNhbGwoTXIsdCk/ci5sZW5ndGg6MDtpLS07KXt2YXIgbj1yW2ldLG89bi5mdW5jO2lmKG51bGw9PW98fG89PWUpcmV0dXJuIG4ubmFtZX1yZXR1cm4gdH1mdW5jdGlvbiBvbyhlKXtyZXR1cm4oQmUuY2FsbChqciwicGxhY2Vob2xkZXIiKT9qcjplKS5wbGFjZWhvbGRlcn1mdW5jdGlvbiBzbygpe3ZhciBlPWpyLml0ZXJhdGVlfHxvYztyZXR1cm4gZT1lPT09b2M/Qmk6ZSxhcmd1bWVudHMubGVuZ3RoP2UoYXJndW1lbnRzWzBdLGFyZ3VtZW50c1sxXSk6ZX1mdW5jdGlvbiBhbyhlLHQpe3ZhciByLGksbj1lLl9fZGF0YV9fO3JldHVybigic3RyaW5nIj09KGk9dHlwZW9mKHI9dCkpfHwibnVtYmVyIj09aXx8InN5bWJvbCI9PWl8fCJib29sZWFuIj09aT8iX19wcm90b19fIiE9PXI6bnVsbD09PXIpP25bInN0cmluZyI9PXR5cGVvZiB0PyJzdHJpbmciOiJoYXNoIl06bi5tYXB9ZnVuY3Rpb24gY28oZSl7Zm9yKHZhciB0PU9hKGUpLHI9dC5sZW5ndGg7ci0tOyl7dmFyIGk9dFtyXSxuPWVbaV07dFtyXT1baSxuLHdvKG4pXX1yZXR1cm4gdH1mdW5jdGlvbiBsbyhlLHQpe3ZhciByPWZ1bmN0aW9uKGUsdCl7cmV0dXJuIG51bGw9PWU/bjplW3RdfShlLHQpO3JldHVybiBPaShyKT9yOm59dmFyIHVvPWhyP2Z1bmN0aW9uKGUpe3JldHVybiBudWxsPT1lP1tdOihlPUxlKGUpLEN0KGhyKGUpLChmdW5jdGlvbih0KXtyZXR1cm4gZXQuY2FsbChlLHQpfSkpKX06dmMsaG89aHI/ZnVuY3Rpb24oZSl7Zm9yKHZhciB0PVtdO2U7KXh0KHQsdW8oZSkpLGU9VmUoZSk7cmV0dXJuIHR9OnZjLGZvPXdpO2Z1bmN0aW9uIF9vKGUsdCxyKXtmb3IodmFyIGk9LTEsbj0odD1nbih0LGUpKS5sZW5ndGgsbz0hMTsrK2k8bjspe3ZhciBzPWpvKHRbaV0pO2lmKCEobz1udWxsIT1lJiZyKGUscykpKWJyZWFrO2U9ZVtzXX1yZXR1cm4gb3x8KytpIT1uP286ISEobj1udWxsPT1lPzA6ZS5sZW5ndGgpJiZlYShuKSYmZ28ocyxuKSYmKEtzKGUpfHx6cyhlKSl9ZnVuY3Rpb24gcG8oZSl7cmV0dXJuImZ1bmN0aW9uIiE9dHlwZW9mIGUuY29uc3RydWN0b3J8fENvKGUpP3t9OkZyKFZlKGUpKX1mdW5jdGlvbiB2byhlKXtyZXR1cm4gS3MoZSl8fHpzKGUpfHwhIShudCYmZSYmZVtudF0pfWZ1bmN0aW9uIGdvKGUsdCl7dmFyIHI9dHlwZW9mIGU7cmV0dXJuISEodD1udWxsPT10P2g6dCkmJigibnVtYmVyIj09cnx8InN5bWJvbCIhPXImJmdlLnRlc3QoZSkpJiZlPi0xJiZlJTE9PTAmJmU8dH1mdW5jdGlvbiB5byhlLHQscil7aWYoIXRhKHIpKXJldHVybiExO3ZhciBpPXR5cGVvZiB0O3JldHVybiEhKCJudW1iZXIiPT1pP0dzKHIpJiZnbyh0LHIubGVuZ3RoKToic3RyaW5nIj09aSYmdCBpbiByKSYmVXMoclt0XSxlKX1mdW5jdGlvbiBtbyhlLHQpe2lmKEtzKGUpKXJldHVybiExO3ZhciByPXR5cGVvZiBlO3JldHVybiEoIm51bWJlciIhPXImJiJzeW1ib2wiIT1yJiYiYm9vbGVhbiIhPXImJm51bGwhPWUmJiFsYShlKSl8fFEudGVzdChlKXx8ISQudGVzdChlKXx8bnVsbCE9dCYmZSBpbiBMZSh0KX1mdW5jdGlvbiBibyhlKXt2YXIgdD1ubyhlKSxyPWpyW3RdO2lmKCJmdW5jdGlvbiIhPXR5cGVvZiByfHwhKHQgaW4gcXIucHJvdG90eXBlKSlyZXR1cm4hMTtpZihlPT09cilyZXR1cm4hMDt2YXIgaT1pbyhyKTtyZXR1cm4hIWkmJmU9PT1pWzBdfShDciYmZm8obmV3IENyKG5ldyBBcnJheUJ1ZmZlcigxKSkpIT1PfHx3ciYmZm8obmV3IHdyKSE9Q3x8THImJmZvKExyLnJlc29sdmUoKSkhPUV8fEVyJiZmbyhuZXcgRXIpIT1BfHx4ciYmZm8obmV3IHhyKSE9UikmJihmbz1mdW5jdGlvbihlKXt2YXIgdD13aShlKSxyPXQ9PUw/ZS5jb25zdHJ1Y3RvcjpuLGk9cj9GbyhyKToiIjtpZihpKXN3aXRjaChpKXtjYXNlIFJyOnJldHVybiBPO2Nhc2UgVHI6cmV0dXJuIEM7Y2FzZSBPcjpyZXR1cm4gRTtjYXNlIEJyOnJldHVybiBBO2Nhc2UgRHI6cmV0dXJuIFJ9cmV0dXJuIHR9KTt2YXIgU289VGU/JHM6Z2M7ZnVuY3Rpb24gQ28oZSl7dmFyIHQ9ZSYmZS5jb25zdHJ1Y3RvcjtyZXR1cm4gZT09PSgiZnVuY3Rpb24iPT10eXBlb2YgdCYmdC5wcm90b3R5cGV8fFJlKX1mdW5jdGlvbiB3byhlKXtyZXR1cm4gZT09ZSYmIXRhKGUpfWZ1bmN0aW9uIExvKGUsdCl7cmV0dXJuIGZ1bmN0aW9uKHIpe3JldHVybiBudWxsIT1yJiZyW2VdPT09dCYmKHQhPT1ufHxlIGluIExlKHIpKX19ZnVuY3Rpb24gRW8oZSx0LHIpe3JldHVybiB0PXZyKHQ9PT1uP2UubGVuZ3RoLTE6dCwwKSxmdW5jdGlvbigpe2Zvcih2YXIgbj1hcmd1bWVudHMsbz0tMSxzPXZyKG4ubGVuZ3RoLXQsMCksYT1pKHMpOysrbzxzOylhW29dPW5bdCtvXTtvPS0xO2Zvcih2YXIgYz1pKHQrMSk7KytvPHQ7KWNbb109bltvXTtyZXR1cm4gY1t0XT1yKGEpLGd0KGUsdGhpcyxjKX19ZnVuY3Rpb24geG8oZSx0KXtyZXR1cm4gdC5sZW5ndGg8Mj9lOlNpKGUsZW4odCwwLC0xKSl9ZnVuY3Rpb24gQW8oZSx0KXtmb3IodmFyIHI9ZS5sZW5ndGgsaT1ncih0Lmxlbmd0aCxyKSxvPUFuKGUpO2ktLTspe3ZhciBzPXRbaV07ZVtpXT1nbyhzLHIpP29bc106bn1yZXR1cm4gZX1mdW5jdGlvbiBrbyhlLHQpe2lmKCgiY29uc3RydWN0b3IiIT09dHx8ImZ1bmN0aW9uIiE9dHlwZW9mIGVbdF0pJiYiX19wcm90b19fIiE9dClyZXR1cm4gZVt0XX12YXIgTW89Qm8oSmkpLFJvPWp0fHxmdW5jdGlvbihlLHQpe3JldHVybiBvdC5zZXRUaW1lb3V0KGUsdCl9LFRvPUJvKCRpKTtmdW5jdGlvbiBPbyhlLHQscil7dmFyIGk9dCsiIjtyZXR1cm4gVG8oZSxmdW5jdGlvbihlLHQpe3ZhciByPXQubGVuZ3RoO2lmKCFyKXJldHVybiBlO3ZhciBpPXItMTtyZXR1cm4gdFtpXT0ocj4xPyImICI6IiIpK3RbaV0sdD10LmpvaW4ocj4yPyIsICI6IiAiKSxlLnJlcGxhY2Uob2UsIntcbi8qIFt3cmFwcGVkIHdpdGggIit0KyJdICovXG4iKX0oaSxmdW5jdGlvbihlLHQpe3JldHVybiBtdChkLChmdW5jdGlvbihyKXt2YXIgaT0iXy4iK3JbMF07dCZyWzFdJiYhd3QoZSxpKSYmZS5wdXNoKGkpfSkpLGUuc29ydCgpfShmdW5jdGlvbihlKXt2YXIgdD1lLm1hdGNoKHNlKTtyZXR1cm4gdD90WzFdLnNwbGl0KGFlKTpbXX0oaSkscikpKX1mdW5jdGlvbiBCbyhlKXt2YXIgdD0wLHI9MDtyZXR1cm4gZnVuY3Rpb24oKXt2YXIgaT15cigpLG89MTYtKGktcik7aWYocj1pLG8+MCl7aWYoKyt0Pj04MDApcmV0dXJuIGFyZ3VtZW50c1swXX1lbHNlIHQ9MDtyZXR1cm4gZS5hcHBseShuLGFyZ3VtZW50cyl9fWZ1bmN0aW9uIERvKGUsdCl7dmFyIHI9LTEsaT1lLmxlbmd0aCxvPWktMTtmb3IodD10PT09bj9pOnQ7KytyPHQ7KXt2YXIgcz1LaShyLG8pLGE9ZVtzXTtlW3NdPWVbcl0sZVtyXT1hfXJldHVybiBlLmxlbmd0aD10LGV9dmFyIFBvLElvLEhvPShQbz1QcygoZnVuY3Rpb24oZSl7dmFyIHQ9W107cmV0dXJuIDQ2PT09ZS5jaGFyQ29kZUF0KDApJiZ0LnB1c2goIiIpLGUucmVwbGFjZShlZSwoZnVuY3Rpb24oZSxyLGksbil7dC5wdXNoKGk/bi5yZXBsYWNlKHVlLCIkMSIpOnJ8fGUpfSkpLHR9KSwoZnVuY3Rpb24oZSl7cmV0dXJuIDUwMD09PUlvLnNpemUmJklvLmNsZWFyKCksZX0pKSxJbz1Qby5jYWNoZSxQbyk7ZnVuY3Rpb24gam8oZSl7aWYoInN0cmluZyI9PXR5cGVvZiBlfHxsYShlKSlyZXR1cm4gZTt2YXIgdD1lKyIiO3JldHVybiIwIj09dCYmMS9lPT0tMS8wPyItMCI6dH1mdW5jdGlvbiBGbyhlKXtpZihudWxsIT1lKXt0cnl7cmV0dXJuIE9lLmNhbGwoZSl9Y2F0Y2goZSl7fXRyeXtyZXR1cm4gZSsiIn1jYXRjaChlKXt9fXJldHVybiIifWZ1bmN0aW9uIFdvKGUpe2lmKGUgaW5zdGFuY2VvZiBxcilyZXR1cm4gZS5jbG9uZSgpO3ZhciB0PW5ldyBVcihlLl9fd3JhcHBlZF9fLGUuX19jaGFpbl9fKTtyZXR1cm4gdC5fX2FjdGlvbnNfXz1BbihlLl9fYWN0aW9uc19fKSx0Ll9faW5kZXhfXz1lLl9faW5kZXhfXyx0Ll9fdmFsdWVzX189ZS5fX3ZhbHVlc19fLHR9dmFyIFVvPUdpKChmdW5jdGlvbihlLHQpe3JldHVybiBZcyhlKT9saShlLHBpKHQsMSxZcywhMCkpOltdfSkpLHFvPUdpKChmdW5jdGlvbihlLHQpe3ZhciByPUpvKHQpO3JldHVybiBZcyhyKSYmKHI9biksWXMoZSk/bGkoZSxwaSh0LDEsWXMsITApLHNvKHIsMikpOltdfSkpLE5vPUdpKChmdW5jdGlvbihlLHQpe3ZhciByPUpvKHQpO3JldHVybiBZcyhyKSYmKHI9biksWXMoZSk/bGkoZSxwaSh0LDEsWXMsITApLG4scik6W119KSk7ZnVuY3Rpb24gem8oZSx0LHIpe3ZhciBpPW51bGw9PWU/MDplLmxlbmd0aDtpZighaSlyZXR1cm4tMTt2YXIgbj1udWxsPT1yPzA6cGEocik7cmV0dXJuIG48MCYmKG49dnIoaStuLDApKSxPdChlLHNvKHQsMyksbil9ZnVuY3Rpb24gS28oZSx0LHIpe3ZhciBpPW51bGw9PWU/MDplLmxlbmd0aDtpZighaSlyZXR1cm4tMTt2YXIgbz1pLTE7cmV0dXJuIHIhPT1uJiYobz1wYShyKSxvPXI8MD92cihpK28sMCk6Z3IobyxpLTEpKSxPdChlLHNvKHQsMyksbywhMCl9ZnVuY3Rpb24gVm8oZSl7cmV0dXJuIG51bGwhPWUmJmUubGVuZ3RoP3BpKGUsMSk6W119ZnVuY3Rpb24gR28oZSl7cmV0dXJuIGUmJmUubGVuZ3RoP2VbMF06bn12YXIgWW89R2koKGZ1bmN0aW9uKGUpe3ZhciB0PUV0KGUscG4pO3JldHVybiB0Lmxlbmd0aCYmdFswXT09PWVbMF0/QWkodCk6W119KSksWG89R2koKGZ1bmN0aW9uKGUpe3ZhciB0PUpvKGUpLHI9RXQoZSxwbik7cmV0dXJuIHQ9PT1KbyhyKT90PW46ci5wb3AoKSxyLmxlbmd0aCYmclswXT09PWVbMF0/QWkocixzbyh0LDIpKTpbXX0pKSxabz1HaSgoZnVuY3Rpb24oZSl7dmFyIHQ9Sm8oZSkscj1FdChlLHBuKTtyZXR1cm4odD0iZnVuY3Rpb24iPT10eXBlb2YgdD90Om4pJiZyLnBvcCgpLHIubGVuZ3RoJiZyWzBdPT09ZVswXT9BaShyLG4sdCk6W119KSk7ZnVuY3Rpb24gSm8oZSl7dmFyIHQ9bnVsbD09ZT8wOmUubGVuZ3RoO3JldHVybiB0P2VbdC0xXTpufXZhciAkbz1HaShRbyk7ZnVuY3Rpb24gUW8oZSx0KXtyZXR1cm4gZSYmZS5sZW5ndGgmJnQmJnQubGVuZ3RoP05pKGUsdCk6ZX12YXIgZXM9ZW8oKGZ1bmN0aW9uKGUsdCl7dmFyIHI9bnVsbD09ZT8wOmUubGVuZ3RoLGk9bmkoZSx0KTtyZXR1cm4gemkoZSxFdCh0LChmdW5jdGlvbihlKXtyZXR1cm4gZ28oZSxyKT8rZTplfSkpLnNvcnQoTG4pKSxpfSkpO2Z1bmN0aW9uIHRzKGUpe3JldHVybiBudWxsPT1lP2U6U3IuY2FsbChlKX12YXIgcnM9R2koKGZ1bmN0aW9uKGUpe3JldHVybiBjbihwaShlLDEsWXMsITApKX0pKSxpcz1HaSgoZnVuY3Rpb24oZSl7dmFyIHQ9Sm8oZSk7cmV0dXJuIFlzKHQpJiYodD1uKSxjbihwaShlLDEsWXMsITApLHNvKHQsMikpfSkpLG5zPUdpKChmdW5jdGlvbihlKXt2YXIgdD1KbyhlKTtyZXR1cm4gdD0iZnVuY3Rpb24iPT10eXBlb2YgdD90Om4sY24ocGkoZSwxLFlzLCEwKSxuLHQpfSkpO2Z1bmN0aW9uIG9zKGUpe2lmKCFlfHwhZS5sZW5ndGgpcmV0dXJuW107dmFyIHQ9MDtyZXR1cm4gZT1DdChlLChmdW5jdGlvbihlKXtpZihZcyhlKSlyZXR1cm4gdD12cihlLmxlbmd0aCx0KSwhMH0pKSxVdCh0LChmdW5jdGlvbih0KXtyZXR1cm4gRXQoZSxIdCh0KSl9KSl9ZnVuY3Rpb24gc3MoZSx0KXtpZighZXx8IWUubGVuZ3RoKXJldHVybltdO3ZhciByPW9zKGUpO3JldHVybiBudWxsPT10P3I6RXQociwoZnVuY3Rpb24oZSl7cmV0dXJuIGd0KHQsbixlKX0pKX12YXIgYXM9R2koKGZ1bmN0aW9uKGUsdCl7cmV0dXJuIFlzKGUpP2xpKGUsdCk6W119KSksY3M9R2koKGZ1bmN0aW9uKGUpe3JldHVybiBfbihDdChlLFlzKSl9KSksbHM9R2koKGZ1bmN0aW9uKGUpe3ZhciB0PUpvKGUpO3JldHVybiBZcyh0KSYmKHQ9biksX24oQ3QoZSxZcyksc28odCwyKSl9KSksdXM9R2koKGZ1bmN0aW9uKGUpe3ZhciB0PUpvKGUpO3JldHVybiB0PSJmdW5jdGlvbiI9PXR5cGVvZiB0P3Q6bixfbihDdChlLFlzKSxuLHQpfSkpLGhzPUdpKG9zKSxmcz1HaSgoZnVuY3Rpb24oZSl7dmFyIHQ9ZS5sZW5ndGgscj10PjE/ZVt0LTFdOm47cmV0dXJuIHI9ImZ1bmN0aW9uIj09dHlwZW9mIHI/KGUucG9wKCkscik6bixzcyhlLHIpfSkpO2Z1bmN0aW9uIF9zKGUpe3ZhciB0PWpyKGUpO3JldHVybiB0Ll9fY2hhaW5fXz0hMCx0fWZ1bmN0aW9uIGRzKGUsdCl7cmV0dXJuIHQoZSl9dmFyIHBzPWVvKChmdW5jdGlvbihlKXt2YXIgdD1lLmxlbmd0aCxyPXQ/ZVswXTowLGk9dGhpcy5fX3dyYXBwZWRfXyxvPWZ1bmN0aW9uKHQpe3JldHVybiBuaSh0LGUpfTtyZXR1cm4hKHQ+MXx8dGhpcy5fX2FjdGlvbnNfXy5sZW5ndGgpJiZpIGluc3RhbmNlb2YgcXImJmdvKHIpPygoaT1pLnNsaWNlKHIsK3IrKHQ/MTowKSkpLl9fYWN0aW9uc19fLnB1c2goe2Z1bmM6ZHMsYXJnczpbb10sdGhpc0FyZzpufSksbmV3IFVyKGksdGhpcy5fX2NoYWluX18pLnRocnUoKGZ1bmN0aW9uKGUpe3JldHVybiB0JiYhZS5sZW5ndGgmJmUucHVzaChuKSxlfSkpKTp0aGlzLnRocnUobyl9KSksdnM9TW4oKGZ1bmN0aW9uKGUsdCxyKXtCZS5jYWxsKGUscik/KytlW3JdOmlpKGUsciwxKX0pKSxncz1Jbih6bykseXM9SW4oS28pO2Z1bmN0aW9uIG1zKGUsdCl7cmV0dXJuKEtzKGUpP210OnVpKShlLHNvKHQsMykpfWZ1bmN0aW9uIGJzKGUsdCl7cmV0dXJuKEtzKGUpP2J0OmhpKShlLHNvKHQsMykpfXZhciBTcz1NbigoZnVuY3Rpb24oZSx0LHIpe0JlLmNhbGwoZSxyKT9lW3JdLnB1c2godCk6aWkoZSxyLFt0XSl9KSksQ3M9R2koKGZ1bmN0aW9uKGUsdCxyKXt2YXIgbj0tMSxvPSJmdW5jdGlvbiI9PXR5cGVvZiB0LHM9R3MoZSk/aShlLmxlbmd0aCk6W107cmV0dXJuIHVpKGUsKGZ1bmN0aW9uKGUpe3NbKytuXT1vP2d0KHQsZSxyKTpraShlLHQscil9KSksc30pKSx3cz1NbigoZnVuY3Rpb24oZSx0LHIpe2lpKGUscix0KX0pKTtmdW5jdGlvbiBMcyhlLHQpe3JldHVybihLcyhlKT9FdDpJaSkoZSxzbyh0LDMpKX12YXIgRXM9TW4oKGZ1bmN0aW9uKGUsdCxyKXtlW3I/MDoxXS5wdXNoKHQpfSksKGZ1bmN0aW9uKCl7cmV0dXJuW1tdLFtdXX0pKSx4cz1HaSgoZnVuY3Rpb24oZSx0KXtpZihudWxsPT1lKXJldHVybltdO3ZhciByPXQubGVuZ3RoO3JldHVybiByPjEmJnlvKGUsdFswXSx0WzFdKT90PVtdOnI+MiYmeW8odFswXSx0WzFdLHRbMl0pJiYodD1bdFswXV0pLFVpKGUscGkodCwxKSxbXSl9KSksQXM9UnR8fGZ1bmN0aW9uKCl7cmV0dXJuIG90LkRhdGUubm93KCl9O2Z1bmN0aW9uIGtzKGUsdCxyKXtyZXR1cm4gdD1yP246dCx0PWUmJm51bGw9PXQ/ZS5sZW5ndGg6dCxYbihlLGwsbixuLG4sbix0KX1mdW5jdGlvbiBNcyhlLHQpe3ZhciByO2lmKCJmdW5jdGlvbiIhPXR5cGVvZiB0KXRocm93IG5ldyBBZShvKTtyZXR1cm4gZT1wYShlKSxmdW5jdGlvbigpe3JldHVybi0tZT4wJiYocj10LmFwcGx5KHRoaXMsYXJndW1lbnRzKSksZTw9MSYmKHQ9bikscn19dmFyIFJzPUdpKChmdW5jdGlvbihlLHQscil7dmFyIGk9MTtpZihyLmxlbmd0aCl7dmFyIG49dHIocixvbyhScykpO2l8PWN9cmV0dXJuIFhuKGUsaSx0LHIsbil9KSksVHM9R2koKGZ1bmN0aW9uKGUsdCxyKXt2YXIgaT0zO2lmKHIubGVuZ3RoKXt2YXIgbj10cihyLG9vKFRzKSk7aXw9Y31yZXR1cm4gWG4odCxpLGUscixuKX0pKTtmdW5jdGlvbiBPcyhlLHQscil7dmFyIGkscyxhLGMsbCx1LGg9MCxmPSExLF89ITEsZD0hMDtpZigiZnVuY3Rpb24iIT10eXBlb2YgZSl0aHJvdyBuZXcgQWUobyk7ZnVuY3Rpb24gcCh0KXt2YXIgcj1pLG89cztyZXR1cm4gaT1zPW4saD10LGM9ZS5hcHBseShvLHIpfWZ1bmN0aW9uIHYoZSl7cmV0dXJuIGg9ZSxsPVJvKHksdCksZj9wKGUpOmN9ZnVuY3Rpb24gZyhlKXt2YXIgcj1lLXU7cmV0dXJuIHU9PT1ufHxyPj10fHxyPDB8fF8mJmUtaD49YX1mdW5jdGlvbiB5KCl7dmFyIGU9QXMoKTtpZihnKGUpKXJldHVybiBtKGUpO2w9Um8oeSxmdW5jdGlvbihlKXt2YXIgcj10LShlLXUpO3JldHVybiBfP2dyKHIsYS0oZS1oKSk6cn0oZSkpfWZ1bmN0aW9uIG0oZSl7cmV0dXJuIGw9bixkJiZpP3AoZSk6KGk9cz1uLGMpfWZ1bmN0aW9uIGIoKXt2YXIgZT1BcygpLHI9ZyhlKTtpZihpPWFyZ3VtZW50cyxzPXRoaXMsdT1lLHIpe2lmKGw9PT1uKXJldHVybiB2KHUpO2lmKF8pcmV0dXJuIGJuKGwpLGw9Um8oeSx0KSxwKHUpfXJldHVybiBsPT09biYmKGw9Um8oeSx0KSksY31yZXR1cm4gdD1nYSh0KXx8MCx0YShyKSYmKGY9ISFyLmxlYWRpbmcsYT0oXz0ibWF4V2FpdCJpbiByKT92cihnYShyLm1heFdhaXQpfHwwLHQpOmEsZD0idHJhaWxpbmciaW4gcj8hIXIudHJhaWxpbmc6ZCksYi5jYW5jZWw9ZnVuY3Rpb24oKXtsIT09biYmYm4obCksaD0wLGk9dT1zPWw9bn0sYi5mbHVzaD1mdW5jdGlvbigpe3JldHVybiBsPT09bj9jOm0oQXMoKSl9LGJ9dmFyIEJzPUdpKChmdW5jdGlvbihlLHQpe3JldHVybiBjaShlLDEsdCl9KSksRHM9R2koKGZ1bmN0aW9uKGUsdCxyKXtyZXR1cm4gY2koZSxnYSh0KXx8MCxyKX0pKTtmdW5jdGlvbiBQcyhlLHQpe2lmKCJmdW5jdGlvbiIhPXR5cGVvZiBlfHxudWxsIT10JiYiZnVuY3Rpb24iIT10eXBlb2YgdCl0aHJvdyBuZXcgQWUobyk7dmFyIHI9ZnVuY3Rpb24oKXt2YXIgaT1hcmd1bWVudHMsbj10P3QuYXBwbHkodGhpcyxpKTppWzBdLG89ci5jYWNoZTtpZihvLmhhcyhuKSlyZXR1cm4gby5nZXQobik7dmFyIHM9ZS5hcHBseSh0aGlzLGkpO3JldHVybiByLmNhY2hlPW8uc2V0KG4scyl8fG8sc307cmV0dXJuIHIuY2FjaGU9bmV3KFBzLkNhY2hlfHxLcikscn1mdW5jdGlvbiBJcyhlKXtpZigiZnVuY3Rpb24iIT10eXBlb2YgZSl0aHJvdyBuZXcgQWUobyk7cmV0dXJuIGZ1bmN0aW9uKCl7dmFyIHQ9YXJndW1lbnRzO3N3aXRjaCh0Lmxlbmd0aCl7Y2FzZSAwOnJldHVybiFlLmNhbGwodGhpcyk7Y2FzZSAxOnJldHVybiFlLmNhbGwodGhpcyx0WzBdKTtjYXNlIDI6cmV0dXJuIWUuY2FsbCh0aGlzLHRbMF0sdFsxXSk7Y2FzZSAzOnJldHVybiFlLmNhbGwodGhpcyx0WzBdLHRbMV0sdFsyXSl9cmV0dXJuIWUuYXBwbHkodGhpcyx0KX19UHMuQ2FjaGU9S3I7dmFyIEhzPXluKChmdW5jdGlvbihlLHQpe3ZhciByPSh0PTE9PXQubGVuZ3RoJiZLcyh0WzBdKT9FdCh0WzBdLE50KHNvKCkpKTpFdChwaSh0LDEpLE50KHNvKCkpKSkubGVuZ3RoO3JldHVybiBHaSgoZnVuY3Rpb24oaSl7Zm9yKHZhciBuPS0xLG89Z3IoaS5sZW5ndGgscik7KytuPG87KWlbbl09dFtuXS5jYWxsKHRoaXMsaVtuXSk7cmV0dXJuIGd0KGUsdGhpcyxpKX0pKX0pKSxqcz1HaSgoZnVuY3Rpb24oZSx0KXt2YXIgcj10cih0LG9vKGpzKSk7cmV0dXJuIFhuKGUsYyxuLHQscil9KSksRnM9R2koKGZ1bmN0aW9uKGUsdCl7dmFyIHI9dHIodCxvbyhGcykpO3JldHVybiBYbihlLDY0LG4sdCxyKX0pKSxXcz1lbygoZnVuY3Rpb24oZSx0KXtyZXR1cm4gWG4oZSwyNTYsbixuLG4sdCl9KSk7ZnVuY3Rpb24gVXMoZSx0KXtyZXR1cm4gZT09PXR8fGUhPWUmJnQhPXR9dmFyIHFzPXpuKExpKSxOcz16bigoZnVuY3Rpb24oZSx0KXtyZXR1cm4gZT49dH0pKSx6cz1NaShmdW5jdGlvbigpe3JldHVybiBhcmd1bWVudHN9KCkpP01pOmZ1bmN0aW9uKGUpe3JldHVybiByYShlKSYmQmUuY2FsbChlLCJjYWxsZWUiKSYmIWV0LmNhbGwoZSwiY2FsbGVlIil9LEtzPWkuaXNBcnJheSxWcz1odD9OdChodCk6ZnVuY3Rpb24oZSl7cmV0dXJuIHJhKGUpJiZ3aShlKT09VH07ZnVuY3Rpb24gR3MoZSl7cmV0dXJuIG51bGwhPWUmJmVhKGUubGVuZ3RoKSYmISRzKGUpfWZ1bmN0aW9uIFlzKGUpe3JldHVybiByYShlKSYmR3MoZSl9dmFyIFhzPWZyfHxnYyxacz1mdD9OdChmdCk6ZnVuY3Rpb24oZSl7cmV0dXJuIHJhKGUpJiZ3aShlKT09eX07ZnVuY3Rpb24gSnMoZSl7aWYoIXJhKGUpKXJldHVybiExO3ZhciB0PXdpKGUpO3JldHVybiB0PT1tfHwiW29iamVjdCBET01FeGNlcHRpb25dIj09dHx8InN0cmluZyI9PXR5cGVvZiBlLm1lc3NhZ2UmJiJzdHJpbmciPT10eXBlb2YgZS5uYW1lJiYhb2EoZSl9ZnVuY3Rpb24gJHMoZSl7aWYoIXRhKGUpKXJldHVybiExO3ZhciB0PXdpKGUpO3JldHVybiB0PT1ifHx0PT1TfHwiW29iamVjdCBBc3luY0Z1bmN0aW9uXSI9PXR8fCJbb2JqZWN0IFByb3h5XSI9PXR9ZnVuY3Rpb24gUXMoZSl7cmV0dXJuIm51bWJlciI9PXR5cGVvZiBlJiZlPT1wYShlKX1mdW5jdGlvbiBlYShlKXtyZXR1cm4ibnVtYmVyIj09dHlwZW9mIGUmJmU+LTEmJmUlMT09MCYmZTw9aH1mdW5jdGlvbiB0YShlKXt2YXIgdD10eXBlb2YgZTtyZXR1cm4gbnVsbCE9ZSYmKCJvYmplY3QiPT10fHwiZnVuY3Rpb24iPT10KX1mdW5jdGlvbiByYShlKXtyZXR1cm4gbnVsbCE9ZSYmIm9iamVjdCI9PXR5cGVvZiBlfXZhciBpYT1fdD9OdChfdCk6ZnVuY3Rpb24oZSl7cmV0dXJuIHJhKGUpJiZmbyhlKT09Q307ZnVuY3Rpb24gbmEoZSl7cmV0dXJuIm51bWJlciI9PXR5cGVvZiBlfHxyYShlKSYmd2koZSk9PXd9ZnVuY3Rpb24gb2EoZSl7aWYoIXJhKGUpfHx3aShlKSE9TClyZXR1cm4hMTt2YXIgdD1WZShlKTtpZihudWxsPT09dClyZXR1cm4hMDt2YXIgcj1CZS5jYWxsKHQsImNvbnN0cnVjdG9yIikmJnQuY29uc3RydWN0b3I7cmV0dXJuImZ1bmN0aW9uIj09dHlwZW9mIHImJnIgaW5zdGFuY2VvZiByJiZPZS5jYWxsKHIpPT1IZX12YXIgc2E9ZHQ/TnQoZHQpOmZ1bmN0aW9uKGUpe3JldHVybiByYShlKSYmd2koZSk9PXh9LGFhPXB0P050KHB0KTpmdW5jdGlvbihlKXtyZXR1cm4gcmEoZSkmJmZvKGUpPT1BfTtmdW5jdGlvbiBjYShlKXtyZXR1cm4ic3RyaW5nIj09dHlwZW9mIGV8fCFLcyhlKSYmcmEoZSkmJndpKGUpPT1rfWZ1bmN0aW9uIGxhKGUpe3JldHVybiJzeW1ib2wiPT10eXBlb2YgZXx8cmEoZSkmJndpKGUpPT1NfXZhciB1YT12dD9OdCh2dCk6ZnVuY3Rpb24oZSl7cmV0dXJuIHJhKGUpJiZlYShlLmxlbmd0aCkmJiEhJGVbd2koZSldfSxoYT16bihQaSksZmE9em4oKGZ1bmN0aW9uKGUsdCl7cmV0dXJuIGU8PXR9KSk7ZnVuY3Rpb24gX2EoZSl7aWYoIWUpcmV0dXJuW107aWYoR3MoZSkpcmV0dXJuIGNhKGUpP29yKGUpOkFuKGUpO2lmKHN0JiZlW3N0XSlyZXR1cm4gZnVuY3Rpb24oZSl7Zm9yKHZhciB0LHI9W107ISh0PWUubmV4dCgpKS5kb25lOylyLnB1c2godC52YWx1ZSk7cmV0dXJuIHJ9KGVbc3RdKCkpO3ZhciB0PWZvKGUpO3JldHVybih0PT1DP1F0OnQ9PUE/cnI6VWEpKGUpfWZ1bmN0aW9uIGRhKGUpe3JldHVybiBlPyhlPWdhKGUpKT09PXV8fGU9PT0tMS8wPzE3OTc2OTMxMzQ4NjIzMTU3ZTI5MiooZTwwPy0xOjEpOmU9PWU/ZTowOjA9PT1lP2U6MH1mdW5jdGlvbiBwYShlKXt2YXIgdD1kYShlKSxyPXQlMTtyZXR1cm4gdD09dD9yP3Qtcjp0OjB9ZnVuY3Rpb24gdmEoZSl7cmV0dXJuIGU/b2kocGEoZSksMCxfKTowfWZ1bmN0aW9uIGdhKGUpe2lmKCJudW1iZXIiPT10eXBlb2YgZSlyZXR1cm4gZTtpZihsYShlKSlyZXR1cm4gZjtpZih0YShlKSl7dmFyIHQ9ImZ1bmN0aW9uIj09dHlwZW9mIGUudmFsdWVPZj9lLnZhbHVlT2YoKTplO2U9dGEodCk/dCsiIjp0fWlmKCJzdHJpbmciIT10eXBlb2YgZSlyZXR1cm4gMD09PWU/ZTorZTtlPXF0KGUpO3ZhciByPWRlLnRlc3QoZSk7cmV0dXJuIHJ8fHZlLnRlc3QoZSk/cnQoZS5zbGljZSgyKSxyPzI6OCk6X2UudGVzdChlKT9mOitlfWZ1bmN0aW9uIHlhKGUpe3JldHVybiBrbihlLEJhKGUpKX1mdW5jdGlvbiBtYShlKXtyZXR1cm4gbnVsbD09ZT8iIjphbihlKX12YXIgYmE9Um4oKGZ1bmN0aW9uKGUsdCl7aWYoQ28odCl8fEdzKHQpKWtuKHQsT2EodCksZSk7ZWxzZSBmb3IodmFyIHIgaW4gdClCZS5jYWxsKHQscikmJlFyKGUscix0W3JdKX0pKSxTYT1SbigoZnVuY3Rpb24oZSx0KXtrbih0LEJhKHQpLGUpfSkpLENhPVJuKChmdW5jdGlvbihlLHQscixpKXtrbih0LEJhKHQpLGUsaSl9KSksd2E9Um4oKGZ1bmN0aW9uKGUsdCxyLGkpe2tuKHQsT2EodCksZSxpKX0pKSxMYT1lbyhuaSksRWE9R2koKGZ1bmN0aW9uKGUsdCl7ZT1MZShlKTt2YXIgcj0tMSxpPXQubGVuZ3RoLG89aT4yP3RbMl06bjtmb3IobyYmeW8odFswXSx0WzFdLG8pJiYoaT0xKTsrK3I8aTspZm9yKHZhciBzPXRbcl0sYT1CYShzKSxjPS0xLGw9YS5sZW5ndGg7KytjPGw7KXt2YXIgdT1hW2NdLGg9ZVt1XTsoaD09PW58fFVzKGgsUmVbdV0pJiYhQmUuY2FsbChlLHUpKSYmKGVbdV09c1t1XSl9cmV0dXJuIGV9KSkseGE9R2koKGZ1bmN0aW9uKGUpe3JldHVybiBlLnB1c2gobixKbiksZ3QoUGEsbixlKX0pKTtmdW5jdGlvbiBBYShlLHQscil7dmFyIGk9bnVsbD09ZT9uOlNpKGUsdCk7cmV0dXJuIGk9PT1uP3I6aX1mdW5jdGlvbiBrYShlLHQpe3JldHVybiBudWxsIT1lJiZfbyhlLHQseGkpfXZhciBNYT1GbigoZnVuY3Rpb24oZSx0LHIpe251bGwhPXQmJiJmdW5jdGlvbiIhPXR5cGVvZiB0LnRvU3RyaW5nJiYodD1JZS5jYWxsKHQpKSxlW3RdPXJ9KSx0YyhuYykpLFJhPUZuKChmdW5jdGlvbihlLHQscil7bnVsbCE9dCYmImZ1bmN0aW9uIiE9dHlwZW9mIHQudG9TdHJpbmcmJih0PUllLmNhbGwodCkpLEJlLmNhbGwoZSx0KT9lW3RdLnB1c2gocik6ZVt0XT1bcl19KSxzbyksVGE9R2koa2kpO2Z1bmN0aW9uIE9hKGUpe3JldHVybiBHcyhlKT9ZcihlKTpEaShlKX1mdW5jdGlvbiBCYShlKXtyZXR1cm4gR3MoZSk/WXIoZSwhMCk6ZnVuY3Rpb24oZSl7aWYoIXRhKGUpKXJldHVybiBmdW5jdGlvbihlKXt2YXIgdD1bXTtpZihudWxsIT1lKWZvcih2YXIgciBpbiBMZShlKSl0LnB1c2gocik7cmV0dXJuIHR9KGUpO3ZhciB0PUNvKGUpLHI9W107Zm9yKHZhciBpIGluIGUpKCJjb25zdHJ1Y3RvciIhPWl8fCF0JiZCZS5jYWxsKGUsaSkpJiZyLnB1c2goaSk7cmV0dXJuIHJ9KGUpfXZhciBEYT1SbigoZnVuY3Rpb24oZSx0LHIpe0ZpKGUsdCxyKX0pKSxQYT1SbigoZnVuY3Rpb24oZSx0LHIsaSl7RmkoZSx0LHIsaSl9KSksSWE9ZW8oKGZ1bmN0aW9uKGUsdCl7dmFyIHI9e307aWYobnVsbD09ZSlyZXR1cm4gcjt2YXIgaT0hMTt0PUV0KHQsKGZ1bmN0aW9uKHQpe3JldHVybiB0PWduKHQsZSksaXx8KGk9dC5sZW5ndGg+MSksdH0pKSxrbihlLHJvKGUpLHIpLGkmJihyPXNpKHIsNywkbikpO2Zvcih2YXIgbj10Lmxlbmd0aDtuLS07KWxuKHIsdFtuXSk7cmV0dXJuIHJ9KSksSGE9ZW8oKGZ1bmN0aW9uKGUsdCl7cmV0dXJuIG51bGw9PWU/e306ZnVuY3Rpb24oZSx0KXtyZXR1cm4gcWkoZSx0LChmdW5jdGlvbih0LHIpe3JldHVybiBrYShlLHIpfSkpfShlLHQpfSkpO2Z1bmN0aW9uIGphKGUsdCl7aWYobnVsbD09ZSlyZXR1cm57fTt2YXIgcj1FdChybyhlKSwoZnVuY3Rpb24oZSl7cmV0dXJuW2VdfSkpO3JldHVybiB0PXNvKHQpLHFpKGUsciwoZnVuY3Rpb24oZSxyKXtyZXR1cm4gdChlLHJbMF0pfSkpfXZhciBGYT1ZbihPYSksV2E9WW4oQmEpO2Z1bmN0aW9uIFVhKGUpe3JldHVybiBudWxsPT1lP1tdOnp0KGUsT2EoZSkpfXZhciBxYT1EbigoZnVuY3Rpb24oZSx0LHIpe3JldHVybiB0PXQudG9Mb3dlckNhc2UoKSxlKyhyP05hKHQpOnQpfSkpO2Z1bmN0aW9uIE5hKGUpe3JldHVybiBKYShtYShlKS50b0xvd2VyQ2FzZSgpKX1mdW5jdGlvbiB6YShlKXtyZXR1cm4oZT1tYShlKSkmJmUucmVwbGFjZSh5ZSxYdCkucmVwbGFjZShLZSwiIil9dmFyIEthPURuKChmdW5jdGlvbihlLHQscil7cmV0dXJuIGUrKHI/Ii0iOiIiKSt0LnRvTG93ZXJDYXNlKCl9KSksVmE9RG4oKGZ1bmN0aW9uKGUsdCxyKXtyZXR1cm4gZSsocj8iICI6IiIpK3QudG9Mb3dlckNhc2UoKX0pKSxHYT1CbigidG9Mb3dlckNhc2UiKSxZYT1EbigoZnVuY3Rpb24oZSx0LHIpe3JldHVybiBlKyhyPyJfIjoiIikrdC50b0xvd2VyQ2FzZSgpfSkpLFhhPURuKChmdW5jdGlvbihlLHQscil7cmV0dXJuIGUrKHI/IiAiOiIiKStKYSh0KX0pKSxaYT1EbigoZnVuY3Rpb24oZSx0LHIpe3JldHVybiBlKyhyPyIgIjoiIikrdC50b1VwcGVyQ2FzZSgpfSkpLEphPUJuKCJ0b1VwcGVyQ2FzZSIpO2Z1bmN0aW9uICRhKGUsdCxyKXtyZXR1cm4gZT1tYShlKSwodD1yP246dCk9PT1uP2Z1bmN0aW9uKGUpe3JldHVybiBYZS50ZXN0KGUpfShlKT9mdW5jdGlvbihlKXtyZXR1cm4gZS5tYXRjaChHZSl8fFtdfShlKTpmdW5jdGlvbihlKXtyZXR1cm4gZS5tYXRjaChjZSl8fFtdfShlKTplLm1hdGNoKHQpfHxbXX12YXIgUWE9R2koKGZ1bmN0aW9uKGUsdCl7dHJ5e3JldHVybiBndChlLG4sdCl9Y2F0Y2goZSl7cmV0dXJuIEpzKGUpP2U6bmV3IFNlKGUpfX0pKSxlYz1lbygoZnVuY3Rpb24oZSx0KXtyZXR1cm4gbXQodCwoZnVuY3Rpb24odCl7dD1qbyh0KSxpaShlLHQsUnMoZVt0XSxlKSl9KSksZX0pKTtmdW5jdGlvbiB0YyhlKXtyZXR1cm4gZnVuY3Rpb24oKXtyZXR1cm4gZX19dmFyIHJjPUhuKCksaWM9SG4oITApO2Z1bmN0aW9uIG5jKGUpe3JldHVybiBlfWZ1bmN0aW9uIG9jKGUpe3JldHVybiBCaSgiZnVuY3Rpb24iPT10eXBlb2YgZT9lOnNpKGUsMSkpfXZhciBzYz1HaSgoZnVuY3Rpb24oZSx0KXtyZXR1cm4gZnVuY3Rpb24ocil7cmV0dXJuIGtpKHIsZSx0KX19KSksYWM9R2koKGZ1bmN0aW9uKGUsdCl7cmV0dXJuIGZ1bmN0aW9uKHIpe3JldHVybiBraShlLHIsdCl9fSkpO2Z1bmN0aW9uIGNjKGUsdCxyKXt2YXIgaT1PYSh0KSxuPWJpKHQsaSk7bnVsbCE9cnx8dGEodCkmJihuLmxlbmd0aHx8IWkubGVuZ3RoKXx8KHI9dCx0PWUsZT10aGlzLG49YmkodCxPYSh0KSkpO3ZhciBvPSEodGEocikmJiJjaGFpbiJpbiByJiYhci5jaGFpbikscz0kcyhlKTtyZXR1cm4gbXQobiwoZnVuY3Rpb24ocil7dmFyIGk9dFtyXTtlW3JdPWkscyYmKGUucHJvdG90eXBlW3JdPWZ1bmN0aW9uKCl7dmFyIHQ9dGhpcy5fX2NoYWluX187aWYob3x8dCl7dmFyIHI9ZSh0aGlzLl9fd3JhcHBlZF9fKSxuPXIuX19hY3Rpb25zX189QW4odGhpcy5fX2FjdGlvbnNfXyk7cmV0dXJuIG4ucHVzaCh7ZnVuYzppLGFyZ3M6YXJndW1lbnRzLHRoaXNBcmc6ZX0pLHIuX19jaGFpbl9fPXQscn1yZXR1cm4gaS5hcHBseShlLHh0KFt0aGlzLnZhbHVlKCldLGFyZ3VtZW50cykpfSl9KSksZX1mdW5jdGlvbiBsYygpe312YXIgdWM9VW4oRXQpLGhjPVVuKFN0KSxmYz1VbihNdCk7ZnVuY3Rpb24gX2MoZSl7cmV0dXJuIG1vKGUpP0h0KGpvKGUpKTpmdW5jdGlvbihlKXtyZXR1cm4gZnVuY3Rpb24odCl7cmV0dXJuIFNpKHQsZSl9fShlKX12YXIgZGM9Tm4oKSxwYz1ObighMCk7ZnVuY3Rpb24gdmMoKXtyZXR1cm5bXX1mdW5jdGlvbiBnYygpe3JldHVybiExfXZhciB5YyxtYz1XbigoZnVuY3Rpb24oZSx0KXtyZXR1cm4gZSt0fSksMCksYmM9Vm4oImNlaWwiKSxTYz1XbigoZnVuY3Rpb24oZSx0KXtyZXR1cm4gZS90fSksMSksQ2M9Vm4oImZsb29yIiksd2M9V24oKGZ1bmN0aW9uKGUsdCl7cmV0dXJuIGUqdH0pLDEpLExjPVZuKCJyb3VuZCIpLEVjPVduKChmdW5jdGlvbihlLHQpe3JldHVybiBlLXR9KSwwKTtyZXR1cm4ganIuYWZ0ZXI9ZnVuY3Rpb24oZSx0KXtpZigiZnVuY3Rpb24iIT10eXBlb2YgdCl0aHJvdyBuZXcgQWUobyk7cmV0dXJuIGU9cGEoZSksZnVuY3Rpb24oKXtpZigtLWU8MSlyZXR1cm4gdC5hcHBseSh0aGlzLGFyZ3VtZW50cyl9fSxqci5hcnk9a3MsanIuYXNzaWduPWJhLGpyLmFzc2lnbkluPVNhLGpyLmFzc2lnbkluV2l0aD1DYSxqci5hc3NpZ25XaXRoPXdhLGpyLmF0PUxhLGpyLmJlZm9yZT1Ncyxqci5iaW5kPVJzLGpyLmJpbmRBbGw9ZWMsanIuYmluZEtleT1Ucyxqci5jYXN0QXJyYXk9ZnVuY3Rpb24oKXtpZighYXJndW1lbnRzLmxlbmd0aClyZXR1cm5bXTt2YXIgZT1hcmd1bWVudHNbMF07cmV0dXJuIEtzKGUpP2U6W2VdfSxqci5jaGFpbj1fcyxqci5jaHVuaz1mdW5jdGlvbihlLHQscil7dD0ocj95byhlLHQscik6dD09PW4pPzE6dnIocGEodCksMCk7dmFyIG89bnVsbD09ZT8wOmUubGVuZ3RoO2lmKCFvfHx0PDEpcmV0dXJuW107Zm9yKHZhciBzPTAsYT0wLGM9aShscihvL3QpKTtzPG87KWNbYSsrXT1lbihlLHMscys9dCk7cmV0dXJuIGN9LGpyLmNvbXBhY3Q9ZnVuY3Rpb24oZSl7Zm9yKHZhciB0PS0xLHI9bnVsbD09ZT8wOmUubGVuZ3RoLGk9MCxuPVtdOysrdDxyOyl7dmFyIG89ZVt0XTtvJiYobltpKytdPW8pfXJldHVybiBufSxqci5jb25jYXQ9ZnVuY3Rpb24oKXt2YXIgZT1hcmd1bWVudHMubGVuZ3RoO2lmKCFlKXJldHVybltdO2Zvcih2YXIgdD1pKGUtMSkscj1hcmd1bWVudHNbMF0sbj1lO24tLTspdFtuLTFdPWFyZ3VtZW50c1tuXTtyZXR1cm4geHQoS3Mocik/QW4ocik6W3JdLHBpKHQsMSkpfSxqci5jb25kPWZ1bmN0aW9uKGUpe3ZhciB0PW51bGw9PWU/MDplLmxlbmd0aCxyPXNvKCk7cmV0dXJuIGU9dD9FdChlLChmdW5jdGlvbihlKXtpZigiZnVuY3Rpb24iIT10eXBlb2YgZVsxXSl0aHJvdyBuZXcgQWUobyk7cmV0dXJuW3IoZVswXSksZVsxXV19KSk6W10sR2koKGZ1bmN0aW9uKHIpe2Zvcih2YXIgaT0tMTsrK2k8dDspe3ZhciBuPWVbaV07aWYoZ3QoblswXSx0aGlzLHIpKXJldHVybiBndChuWzFdLHRoaXMscil9fSkpfSxqci5jb25mb3Jtcz1mdW5jdGlvbihlKXtyZXR1cm4gZnVuY3Rpb24oZSl7dmFyIHQ9T2EoZSk7cmV0dXJuIGZ1bmN0aW9uKHIpe3JldHVybiBhaShyLGUsdCl9fShzaShlLDEpKX0sanIuY29uc3RhbnQ9dGMsanIuY291bnRCeT12cyxqci5jcmVhdGU9ZnVuY3Rpb24oZSx0KXt2YXIgcj1GcihlKTtyZXR1cm4gbnVsbD09dD9yOnJpKHIsdCl9LGpyLmN1cnJ5PWZ1bmN0aW9uIGUodCxyLGkpe3ZhciBvPVhuKHQsOCxuLG4sbixuLG4scj1pP246cik7cmV0dXJuIG8ucGxhY2Vob2xkZXI9ZS5wbGFjZWhvbGRlcixvfSxqci5jdXJyeVJpZ2h0PWZ1bmN0aW9uIGUodCxyLGkpe3ZhciBvPVhuKHQsMTYsbixuLG4sbixuLHI9aT9uOnIpO3JldHVybiBvLnBsYWNlaG9sZGVyPWUucGxhY2Vob2xkZXIsb30sanIuZGVib3VuY2U9T3MsanIuZGVmYXVsdHM9RWEsanIuZGVmYXVsdHNEZWVwPXhhLGpyLmRlZmVyPUJzLGpyLmRlbGF5PURzLGpyLmRpZmZlcmVuY2U9VW8sanIuZGlmZmVyZW5jZUJ5PXFvLGpyLmRpZmZlcmVuY2VXaXRoPU5vLGpyLmRyb3A9ZnVuY3Rpb24oZSx0LHIpe3ZhciBpPW51bGw9PWU/MDplLmxlbmd0aDtyZXR1cm4gaT9lbihlLCh0PXJ8fHQ9PT1uPzE6cGEodCkpPDA/MDp0LGkpOltdfSxqci5kcm9wUmlnaHQ9ZnVuY3Rpb24oZSx0LHIpe3ZhciBpPW51bGw9PWU/MDplLmxlbmd0aDtyZXR1cm4gaT9lbihlLDAsKHQ9aS0odD1yfHx0PT09bj8xOnBhKHQpKSk8MD8wOnQpOltdfSxqci5kcm9wUmlnaHRXaGlsZT1mdW5jdGlvbihlLHQpe3JldHVybiBlJiZlLmxlbmd0aD9obihlLHNvKHQsMyksITAsITApOltdfSxqci5kcm9wV2hpbGU9ZnVuY3Rpb24oZSx0KXtyZXR1cm4gZSYmZS5sZW5ndGg/aG4oZSxzbyh0LDMpLCEwKTpbXX0sanIuZmlsbD1mdW5jdGlvbihlLHQscixpKXt2YXIgbz1udWxsPT1lPzA6ZS5sZW5ndGg7cmV0dXJuIG8/KHImJiJudW1iZXIiIT10eXBlb2YgciYmeW8oZSx0LHIpJiYocj0wLGk9byksZnVuY3Rpb24oZSx0LHIsaSl7dmFyIG89ZS5sZW5ndGg7Zm9yKChyPXBhKHIpKTwwJiYocj0tcj5vPzA6bytyKSwoaT1pPT09bnx8aT5vP286cGEoaSkpPDAmJihpKz1vKSxpPXI+aT8wOnZhKGkpO3I8aTspZVtyKytdPXQ7cmV0dXJuIGV9KGUsdCxyLGkpKTpbXX0sanIuZmlsdGVyPWZ1bmN0aW9uKGUsdCl7cmV0dXJuKEtzKGUpP0N0OmRpKShlLHNvKHQsMykpfSxqci5mbGF0TWFwPWZ1bmN0aW9uKGUsdCl7cmV0dXJuIHBpKExzKGUsdCksMSl9LGpyLmZsYXRNYXBEZWVwPWZ1bmN0aW9uKGUsdCl7cmV0dXJuIHBpKExzKGUsdCksdSl9LGpyLmZsYXRNYXBEZXB0aD1mdW5jdGlvbihlLHQscil7cmV0dXJuIHI9cj09PW4/MTpwYShyKSxwaShMcyhlLHQpLHIpfSxqci5mbGF0dGVuPVZvLGpyLmZsYXR0ZW5EZWVwPWZ1bmN0aW9uKGUpe3JldHVybiBudWxsIT1lJiZlLmxlbmd0aD9waShlLHUpOltdfSxqci5mbGF0dGVuRGVwdGg9ZnVuY3Rpb24oZSx0KXtyZXR1cm4gbnVsbCE9ZSYmZS5sZW5ndGg/cGkoZSx0PXQ9PT1uPzE6cGEodCkpOltdfSxqci5mbGlwPWZ1bmN0aW9uKGUpe3JldHVybiBYbihlLDUxMil9LGpyLmZsb3c9cmMsanIuZmxvd1JpZ2h0PWljLGpyLmZyb21QYWlycz1mdW5jdGlvbihlKXtmb3IodmFyIHQ9LTEscj1udWxsPT1lPzA6ZS5sZW5ndGgsaT17fTsrK3Q8cjspe3ZhciBuPWVbdF07aVtuWzBdXT1uWzFdfXJldHVybiBpfSxqci5mdW5jdGlvbnM9ZnVuY3Rpb24oZSl7cmV0dXJuIG51bGw9PWU/W106YmkoZSxPYShlKSl9LGpyLmZ1bmN0aW9uc0luPWZ1bmN0aW9uKGUpe3JldHVybiBudWxsPT1lP1tdOmJpKGUsQmEoZSkpfSxqci5ncm91cEJ5PVNzLGpyLmluaXRpYWw9ZnVuY3Rpb24oZSl7cmV0dXJuIG51bGwhPWUmJmUubGVuZ3RoP2VuKGUsMCwtMSk6W119LGpyLmludGVyc2VjdGlvbj1Zbyxqci5pbnRlcnNlY3Rpb25CeT1Ybyxqci5pbnRlcnNlY3Rpb25XaXRoPVpvLGpyLmludmVydD1NYSxqci5pbnZlcnRCeT1SYSxqci5pbnZva2VNYXA9Q3MsanIuaXRlcmF0ZWU9b2MsanIua2V5Qnk9d3MsanIua2V5cz1PYSxqci5rZXlzSW49QmEsanIubWFwPUxzLGpyLm1hcEtleXM9ZnVuY3Rpb24oZSx0KXt2YXIgcj17fTtyZXR1cm4gdD1zbyh0LDMpLHlpKGUsKGZ1bmN0aW9uKGUsaSxuKXtpaShyLHQoZSxpLG4pLGUpfSkpLHJ9LGpyLm1hcFZhbHVlcz1mdW5jdGlvbihlLHQpe3ZhciByPXt9O3JldHVybiB0PXNvKHQsMykseWkoZSwoZnVuY3Rpb24oZSxpLG4pe2lpKHIsaSx0KGUsaSxuKSl9KSkscn0sanIubWF0Y2hlcz1mdW5jdGlvbihlKXtyZXR1cm4gSGkoc2koZSwxKSl9LGpyLm1hdGNoZXNQcm9wZXJ0eT1mdW5jdGlvbihlLHQpe3JldHVybiBqaShlLHNpKHQsMSkpfSxqci5tZW1vaXplPVBzLGpyLm1lcmdlPURhLGpyLm1lcmdlV2l0aD1QYSxqci5tZXRob2Q9c2MsanIubWV0aG9kT2Y9YWMsanIubWl4aW49Y2MsanIubmVnYXRlPUlzLGpyLm50aEFyZz1mdW5jdGlvbihlKXtyZXR1cm4gZT1wYShlKSxHaSgoZnVuY3Rpb24odCl7cmV0dXJuIFdpKHQsZSl9KSl9LGpyLm9taXQ9SWEsanIub21pdEJ5PWZ1bmN0aW9uKGUsdCl7cmV0dXJuIGphKGUsSXMoc28odCkpKX0sanIub25jZT1mdW5jdGlvbihlKXtyZXR1cm4gTXMoMixlKX0sanIub3JkZXJCeT1mdW5jdGlvbihlLHQscixpKXtyZXR1cm4gbnVsbD09ZT9bXTooS3ModCl8fCh0PW51bGw9PXQ/W106W3RdKSxLcyhyPWk/bjpyKXx8KHI9bnVsbD09cj9bXTpbcl0pLFVpKGUsdCxyKSl9LGpyLm92ZXI9dWMsanIub3ZlckFyZ3M9SHMsanIub3ZlckV2ZXJ5PWhjLGpyLm92ZXJTb21lPWZjLGpyLnBhcnRpYWw9anMsanIucGFydGlhbFJpZ2h0PUZzLGpyLnBhcnRpdGlvbj1Fcyxqci5waWNrPUhhLGpyLnBpY2tCeT1qYSxqci5wcm9wZXJ0eT1fYyxqci5wcm9wZXJ0eU9mPWZ1bmN0aW9uKGUpe3JldHVybiBmdW5jdGlvbih0KXtyZXR1cm4gbnVsbD09ZT9uOlNpKGUsdCl9fSxqci5wdWxsPSRvLGpyLnB1bGxBbGw9UW8sanIucHVsbEFsbEJ5PWZ1bmN0aW9uKGUsdCxyKXtyZXR1cm4gZSYmZS5sZW5ndGgmJnQmJnQubGVuZ3RoP05pKGUsdCxzbyhyLDIpKTplfSxqci5wdWxsQWxsV2l0aD1mdW5jdGlvbihlLHQscil7cmV0dXJuIGUmJmUubGVuZ3RoJiZ0JiZ0Lmxlbmd0aD9OaShlLHQsbixyKTplfSxqci5wdWxsQXQ9ZXMsanIucmFuZ2U9ZGMsanIucmFuZ2VSaWdodD1wYyxqci5yZWFyZz1Xcyxqci5yZWplY3Q9ZnVuY3Rpb24oZSx0KXtyZXR1cm4oS3MoZSk/Q3Q6ZGkpKGUsSXMoc28odCwzKSkpfSxqci5yZW1vdmU9ZnVuY3Rpb24oZSx0KXt2YXIgcj1bXTtpZighZXx8IWUubGVuZ3RoKXJldHVybiByO3ZhciBpPS0xLG49W10sbz1lLmxlbmd0aDtmb3IodD1zbyh0LDMpOysraTxvOyl7dmFyIHM9ZVtpXTt0KHMsaSxlKSYmKHIucHVzaChzKSxuLnB1c2goaSkpfXJldHVybiB6aShlLG4pLHJ9LGpyLnJlc3Q9ZnVuY3Rpb24oZSx0KXtpZigiZnVuY3Rpb24iIT10eXBlb2YgZSl0aHJvdyBuZXcgQWUobyk7cmV0dXJuIEdpKGUsdD10PT09bj90OnBhKHQpKX0sanIucmV2ZXJzZT10cyxqci5zYW1wbGVTaXplPWZ1bmN0aW9uKGUsdCxyKXtyZXR1cm4gdD0ocj95byhlLHQscik6dD09PW4pPzE6cGEodCksKEtzKGUpP1pyOlhpKShlLHQpfSxqci5zZXQ9ZnVuY3Rpb24oZSx0LHIpe3JldHVybiBudWxsPT1lP2U6WmkoZSx0LHIpfSxqci5zZXRXaXRoPWZ1bmN0aW9uKGUsdCxyLGkpe3JldHVybiBpPSJmdW5jdGlvbiI9PXR5cGVvZiBpP2k6bixudWxsPT1lP2U6WmkoZSx0LHIsaSl9LGpyLnNodWZmbGU9ZnVuY3Rpb24oZSl7cmV0dXJuKEtzKGUpP0pyOlFpKShlKX0sanIuc2xpY2U9ZnVuY3Rpb24oZSx0LHIpe3ZhciBpPW51bGw9PWU/MDplLmxlbmd0aDtyZXR1cm4gaT8ociYmIm51bWJlciIhPXR5cGVvZiByJiZ5byhlLHQscik/KHQ9MCxyPWkpOih0PW51bGw9PXQ/MDpwYSh0KSxyPXI9PT1uP2k6cGEocikpLGVuKGUsdCxyKSk6W119LGpyLnNvcnRCeT14cyxqci5zb3J0ZWRVbmlxPWZ1bmN0aW9uKGUpe3JldHVybiBlJiZlLmxlbmd0aD9vbihlKTpbXX0sanIuc29ydGVkVW5pcUJ5PWZ1bmN0aW9uKGUsdCl7cmV0dXJuIGUmJmUubGVuZ3RoP29uKGUsc28odCwyKSk6W119LGpyLnNwbGl0PWZ1bmN0aW9uKGUsdCxyKXtyZXR1cm4gciYmIm51bWJlciIhPXR5cGVvZiByJiZ5byhlLHQscikmJih0PXI9biksKHI9cj09PW4/XzpyPj4+MCk/KGU9bWEoZSkpJiYoInN0cmluZyI9PXR5cGVvZiB0fHxudWxsIT10JiYhc2EodCkpJiYhKHQ9YW4odCkpJiYkdChlKT9tbihvcihlKSwwLHIpOmUuc3BsaXQodCxyKTpbXX0sanIuc3ByZWFkPWZ1bmN0aW9uKGUsdCl7aWYoImZ1bmN0aW9uIiE9dHlwZW9mIGUpdGhyb3cgbmV3IEFlKG8pO3JldHVybiB0PW51bGw9PXQ/MDp2cihwYSh0KSwwKSxHaSgoZnVuY3Rpb24ocil7dmFyIGk9clt0XSxuPW1uKHIsMCx0KTtyZXR1cm4gaSYmeHQobixpKSxndChlLHRoaXMsbil9KSl9LGpyLnRhaWw9ZnVuY3Rpb24oZSl7dmFyIHQ9bnVsbD09ZT8wOmUubGVuZ3RoO3JldHVybiB0P2VuKGUsMSx0KTpbXX0sanIudGFrZT1mdW5jdGlvbihlLHQscil7cmV0dXJuIGUmJmUubGVuZ3RoP2VuKGUsMCwodD1yfHx0PT09bj8xOnBhKHQpKTwwPzA6dCk6W119LGpyLnRha2VSaWdodD1mdW5jdGlvbihlLHQscil7dmFyIGk9bnVsbD09ZT8wOmUubGVuZ3RoO3JldHVybiBpP2VuKGUsKHQ9aS0odD1yfHx0PT09bj8xOnBhKHQpKSk8MD8wOnQsaSk6W119LGpyLnRha2VSaWdodFdoaWxlPWZ1bmN0aW9uKGUsdCl7cmV0dXJuIGUmJmUubGVuZ3RoP2huKGUsc28odCwzKSwhMSwhMCk6W119LGpyLnRha2VXaGlsZT1mdW5jdGlvbihlLHQpe3JldHVybiBlJiZlLmxlbmd0aD9obihlLHNvKHQsMykpOltdfSxqci50YXA9ZnVuY3Rpb24oZSx0KXtyZXR1cm4gdChlKSxlfSxqci50aHJvdHRsZT1mdW5jdGlvbihlLHQscil7dmFyIGk9ITAsbj0hMDtpZigiZnVuY3Rpb24iIT10eXBlb2YgZSl0aHJvdyBuZXcgQWUobyk7cmV0dXJuIHRhKHIpJiYoaT0ibGVhZGluZyJpbiByPyEhci5sZWFkaW5nOmksbj0idHJhaWxpbmciaW4gcj8hIXIudHJhaWxpbmc6biksT3MoZSx0LHtsZWFkaW5nOmksbWF4V2FpdDp0LHRyYWlsaW5nOm59KX0sanIudGhydT1kcyxqci50b0FycmF5PV9hLGpyLnRvUGFpcnM9RmEsanIudG9QYWlyc0luPVdhLGpyLnRvUGF0aD1mdW5jdGlvbihlKXtyZXR1cm4gS3MoZSk/RXQoZSxqbyk6bGEoZSk/W2VdOkFuKEhvKG1hKGUpKSl9LGpyLnRvUGxhaW5PYmplY3Q9eWEsanIudHJhbnNmb3JtPWZ1bmN0aW9uKGUsdCxyKXt2YXIgaT1LcyhlKSxuPWl8fFhzKGUpfHx1YShlKTtpZih0PXNvKHQsNCksbnVsbD09cil7dmFyIG89ZSYmZS5jb25zdHJ1Y3RvcjtyPW4/aT9uZXcgbzpbXTp0YShlKSYmJHMobyk/RnIoVmUoZSkpOnt9fXJldHVybihuP210OnlpKShlLChmdW5jdGlvbihlLGksbil7cmV0dXJuIHQocixlLGksbil9KSkscn0sanIudW5hcnk9ZnVuY3Rpb24oZSl7cmV0dXJuIGtzKGUsMSl9LGpyLnVuaW9uPXJzLGpyLnVuaW9uQnk9aXMsanIudW5pb25XaXRoPW5zLGpyLnVuaXE9ZnVuY3Rpb24oZSl7cmV0dXJuIGUmJmUubGVuZ3RoP2NuKGUpOltdfSxqci51bmlxQnk9ZnVuY3Rpb24oZSx0KXtyZXR1cm4gZSYmZS5sZW5ndGg/Y24oZSxzbyh0LDIpKTpbXX0sanIudW5pcVdpdGg9ZnVuY3Rpb24oZSx0KXtyZXR1cm4gdD0iZnVuY3Rpb24iPT10eXBlb2YgdD90Om4sZSYmZS5sZW5ndGg/Y24oZSxuLHQpOltdfSxqci51bnNldD1mdW5jdGlvbihlLHQpe3JldHVybiBudWxsPT1lfHxsbihlLHQpfSxqci51bnppcD1vcyxqci51bnppcFdpdGg9c3MsanIudXBkYXRlPWZ1bmN0aW9uKGUsdCxyKXtyZXR1cm4gbnVsbD09ZT9lOnVuKGUsdCx2bihyKSl9LGpyLnVwZGF0ZVdpdGg9ZnVuY3Rpb24oZSx0LHIsaSl7cmV0dXJuIGk9ImZ1bmN0aW9uIj09dHlwZW9mIGk/aTpuLG51bGw9PWU/ZTp1bihlLHQsdm4ociksaSl9LGpyLnZhbHVlcz1VYSxqci52YWx1ZXNJbj1mdW5jdGlvbihlKXtyZXR1cm4gbnVsbD09ZT9bXTp6dChlLEJhKGUpKX0sanIud2l0aG91dD1hcyxqci53b3Jkcz0kYSxqci53cmFwPWZ1bmN0aW9uKGUsdCl7cmV0dXJuIGpzKHZuKHQpLGUpfSxqci54b3I9Y3MsanIueG9yQnk9bHMsanIueG9yV2l0aD11cyxqci56aXA9aHMsanIuemlwT2JqZWN0PWZ1bmN0aW9uKGUsdCl7cmV0dXJuIGRuKGV8fFtdLHR8fFtdLFFyKX0sanIuemlwT2JqZWN0RGVlcD1mdW5jdGlvbihlLHQpe3JldHVybiBkbihlfHxbXSx0fHxbXSxaaSl9LGpyLnppcFdpdGg9ZnMsanIuZW50cmllcz1GYSxqci5lbnRyaWVzSW49V2EsanIuZXh0ZW5kPVNhLGpyLmV4dGVuZFdpdGg9Q2EsY2MoanIsanIpLGpyLmFkZD1tYyxqci5hdHRlbXB0PVFhLGpyLmNhbWVsQ2FzZT1xYSxqci5jYXBpdGFsaXplPU5hLGpyLmNlaWw9YmMsanIuY2xhbXA9ZnVuY3Rpb24oZSx0LHIpe3JldHVybiByPT09biYmKHI9dCx0PW4pLHIhPT1uJiYocj0ocj1nYShyKSk9PXI/cjowKSx0IT09biYmKHQ9KHQ9Z2EodCkpPT10P3Q6MCksb2koZ2EoZSksdCxyKX0sanIuY2xvbmU9ZnVuY3Rpb24oZSl7cmV0dXJuIHNpKGUsNCl9LGpyLmNsb25lRGVlcD1mdW5jdGlvbihlKXtyZXR1cm4gc2koZSw1KX0sanIuY2xvbmVEZWVwV2l0aD1mdW5jdGlvbihlLHQpe3JldHVybiBzaShlLDUsdD0iZnVuY3Rpb24iPT10eXBlb2YgdD90Om4pfSxqci5jbG9uZVdpdGg9ZnVuY3Rpb24oZSx0KXtyZXR1cm4gc2koZSw0LHQ9ImZ1bmN0aW9uIj09dHlwZW9mIHQ/dDpuKX0sanIuY29uZm9ybXNUbz1mdW5jdGlvbihlLHQpe3JldHVybiBudWxsPT10fHxhaShlLHQsT2EodCkpfSxqci5kZWJ1cnI9emEsanIuZGVmYXVsdFRvPWZ1bmN0aW9uKGUsdCl7cmV0dXJuIG51bGw9PWV8fGUhPWU/dDplfSxqci5kaXZpZGU9U2MsanIuZW5kc1dpdGg9ZnVuY3Rpb24oZSx0LHIpe2U9bWEoZSksdD1hbih0KTt2YXIgaT1lLmxlbmd0aCxvPXI9cj09PW4/aTpvaShwYShyKSwwLGkpO3JldHVybihyLT10Lmxlbmd0aCk+PTAmJmUuc2xpY2UocixvKT09dH0sanIuZXE9VXMsanIuZXNjYXBlPWZ1bmN0aW9uKGUpe3JldHVybihlPW1hKGUpKSYmWS50ZXN0KGUpP2UucmVwbGFjZShWLFp0KTplfSxqci5lc2NhcGVSZWdFeHA9ZnVuY3Rpb24oZSl7cmV0dXJuKGU9bWEoZSkpJiZyZS50ZXN0KGUpP2UucmVwbGFjZSh0ZSwiXFwkJiIpOmV9LGpyLmV2ZXJ5PWZ1bmN0aW9uKGUsdCxyKXt2YXIgaT1LcyhlKT9TdDpmaTtyZXR1cm4gciYmeW8oZSx0LHIpJiYodD1uKSxpKGUsc28odCwzKSl9LGpyLmZpbmQ9Z3MsanIuZmluZEluZGV4PXpvLGpyLmZpbmRLZXk9ZnVuY3Rpb24oZSx0KXtyZXR1cm4gVHQoZSxzbyh0LDMpLHlpKX0sanIuZmluZExhc3Q9eXMsanIuZmluZExhc3RJbmRleD1Lbyxqci5maW5kTGFzdEtleT1mdW5jdGlvbihlLHQpe3JldHVybiBUdChlLHNvKHQsMyksbWkpfSxqci5mbG9vcj1DYyxqci5mb3JFYWNoPW1zLGpyLmZvckVhY2hSaWdodD1icyxqci5mb3JJbj1mdW5jdGlvbihlLHQpe3JldHVybiBudWxsPT1lP2U6dmkoZSxzbyh0LDMpLEJhKX0sanIuZm9ySW5SaWdodD1mdW5jdGlvbihlLHQpe3JldHVybiBudWxsPT1lP2U6Z2koZSxzbyh0LDMpLEJhKX0sanIuZm9yT3duPWZ1bmN0aW9uKGUsdCl7cmV0dXJuIGUmJnlpKGUsc28odCwzKSl9LGpyLmZvck93blJpZ2h0PWZ1bmN0aW9uKGUsdCl7cmV0dXJuIGUmJm1pKGUsc28odCwzKSl9LGpyLmdldD1BYSxqci5ndD1xcyxqci5ndGU9TnMsanIuaGFzPWZ1bmN0aW9uKGUsdCl7cmV0dXJuIG51bGwhPWUmJl9vKGUsdCxFaSl9LGpyLmhhc0luPWthLGpyLmhlYWQ9R28sanIuaWRlbnRpdHk9bmMsanIuaW5jbHVkZXM9ZnVuY3Rpb24oZSx0LHIsaSl7ZT1HcyhlKT9lOlVhKGUpLHI9ciYmIWk/cGEocik6MDt2YXIgbj1lLmxlbmd0aDtyZXR1cm4gcjwwJiYocj12cihuK3IsMCkpLGNhKGUpP3I8PW4mJmUuaW5kZXhPZih0LHIpPi0xOiEhbiYmQnQoZSx0LHIpPi0xfSxqci5pbmRleE9mPWZ1bmN0aW9uKGUsdCxyKXt2YXIgaT1udWxsPT1lPzA6ZS5sZW5ndGg7aWYoIWkpcmV0dXJuLTE7dmFyIG49bnVsbD09cj8wOnBhKHIpO3JldHVybiBuPDAmJihuPXZyKGkrbiwwKSksQnQoZSx0LG4pfSxqci5pblJhbmdlPWZ1bmN0aW9uKGUsdCxyKXtyZXR1cm4gdD1kYSh0KSxyPT09bj8ocj10LHQ9MCk6cj1kYShyKSxmdW5jdGlvbihlLHQscil7cmV0dXJuIGU+PWdyKHQscikmJmU8dnIodCxyKX0oZT1nYShlKSx0LHIpfSxqci5pbnZva2U9VGEsanIuaXNBcmd1bWVudHM9enMsanIuaXNBcnJheT1Lcyxqci5pc0FycmF5QnVmZmVyPVZzLGpyLmlzQXJyYXlMaWtlPUdzLGpyLmlzQXJyYXlMaWtlT2JqZWN0PVlzLGpyLmlzQm9vbGVhbj1mdW5jdGlvbihlKXtyZXR1cm4hMD09PWV8fCExPT09ZXx8cmEoZSkmJndpKGUpPT1nfSxqci5pc0J1ZmZlcj1Ycyxqci5pc0RhdGU9WnMsanIuaXNFbGVtZW50PWZ1bmN0aW9uKGUpe3JldHVybiByYShlKSYmMT09PWUubm9kZVR5cGUmJiFvYShlKX0sanIuaXNFbXB0eT1mdW5jdGlvbihlKXtpZihudWxsPT1lKXJldHVybiEwO2lmKEdzKGUpJiYoS3MoZSl8fCJzdHJpbmciPT10eXBlb2YgZXx8ImZ1bmN0aW9uIj09dHlwZW9mIGUuc3BsaWNlfHxYcyhlKXx8dWEoZSl8fHpzKGUpKSlyZXR1cm4hZS5sZW5ndGg7dmFyIHQ9Zm8oZSk7aWYodD09Q3x8dD09QSlyZXR1cm4hZS5zaXplO2lmKENvKGUpKXJldHVybiFEaShlKS5sZW5ndGg7Zm9yKHZhciByIGluIGUpaWYoQmUuY2FsbChlLHIpKXJldHVybiExO3JldHVybiEwfSxqci5pc0VxdWFsPWZ1bmN0aW9uKGUsdCl7cmV0dXJuIFJpKGUsdCl9LGpyLmlzRXF1YWxXaXRoPWZ1bmN0aW9uKGUsdCxyKXt2YXIgaT0ocj0iZnVuY3Rpb24iPT10eXBlb2Ygcj9yOm4pP3IoZSx0KTpuO3JldHVybiBpPT09bj9SaShlLHQsbixyKTohIWl9LGpyLmlzRXJyb3I9SnMsanIuaXNGaW5pdGU9ZnVuY3Rpb24oZSl7cmV0dXJuIm51bWJlciI9PXR5cGVvZiBlJiZfcihlKX0sanIuaXNGdW5jdGlvbj0kcyxqci5pc0ludGVnZXI9UXMsanIuaXNMZW5ndGg9ZWEsanIuaXNNYXA9aWEsanIuaXNNYXRjaD1mdW5jdGlvbihlLHQpe3JldHVybiBlPT09dHx8VGkoZSx0LGNvKHQpKX0sanIuaXNNYXRjaFdpdGg9ZnVuY3Rpb24oZSx0LHIpe3JldHVybiByPSJmdW5jdGlvbiI9PXR5cGVvZiByP3I6bixUaShlLHQsY28odCkscil9LGpyLmlzTmFOPWZ1bmN0aW9uKGUpe3JldHVybiBuYShlKSYmZSE9K2V9LGpyLmlzTmF0aXZlPWZ1bmN0aW9uKGUpe2lmKFNvKGUpKXRocm93IG5ldyBTZSgiVW5zdXBwb3J0ZWQgY29yZS1qcyB1c2UuIFRyeSBodHRwczovL25wbXMuaW8vc2VhcmNoP3E9cG9ueWZpbGwuIik7cmV0dXJuIE9pKGUpfSxqci5pc05pbD1mdW5jdGlvbihlKXtyZXR1cm4gbnVsbD09ZX0sanIuaXNOdWxsPWZ1bmN0aW9uKGUpe3JldHVybiBudWxsPT09ZX0sanIuaXNOdW1iZXI9bmEsanIuaXNPYmplY3Q9dGEsanIuaXNPYmplY3RMaWtlPXJhLGpyLmlzUGxhaW5PYmplY3Q9b2EsanIuaXNSZWdFeHA9c2EsanIuaXNTYWZlSW50ZWdlcj1mdW5jdGlvbihlKXtyZXR1cm4gUXMoZSkmJmU+PS05MDA3MTk5MjU0NzQwOTkxJiZlPD1ofSxqci5pc1NldD1hYSxqci5pc1N0cmluZz1jYSxqci5pc1N5bWJvbD1sYSxqci5pc1R5cGVkQXJyYXk9dWEsanIuaXNVbmRlZmluZWQ9ZnVuY3Rpb24oZSl7cmV0dXJuIGU9PT1ufSxqci5pc1dlYWtNYXA9ZnVuY3Rpb24oZSl7cmV0dXJuIHJhKGUpJiZmbyhlKT09Un0sanIuaXNXZWFrU2V0PWZ1bmN0aW9uKGUpe3JldHVybiByYShlKSYmIltvYmplY3QgV2Vha1NldF0iPT13aShlKX0sanIuam9pbj1mdW5jdGlvbihlLHQpe3JldHVybiBudWxsPT1lPyIiOmRyLmNhbGwoZSx0KX0sanIua2ViYWJDYXNlPUthLGpyLmxhc3Q9Sm8sanIubGFzdEluZGV4T2Y9ZnVuY3Rpb24oZSx0LHIpe3ZhciBpPW51bGw9PWU/MDplLmxlbmd0aDtpZighaSlyZXR1cm4tMTt2YXIgbz1pO3JldHVybiByIT09biYmKG89KG89cGEocikpPDA/dnIoaStvLDApOmdyKG8saS0xKSksdD09dD9mdW5jdGlvbihlLHQscil7Zm9yKHZhciBpPXIrMTtpLS07KWlmKGVbaV09PT10KXJldHVybiBpO3JldHVybiBpfShlLHQsbyk6T3QoZSxQdCxvLCEwKX0sanIubG93ZXJDYXNlPVZhLGpyLmxvd2VyRmlyc3Q9R2EsanIubHQ9aGEsanIubHRlPWZhLGpyLm1heD1mdW5jdGlvbihlKXtyZXR1cm4gZSYmZS5sZW5ndGg/X2koZSxuYyxMaSk6bn0sanIubWF4Qnk9ZnVuY3Rpb24oZSx0KXtyZXR1cm4gZSYmZS5sZW5ndGg/X2koZSxzbyh0LDIpLExpKTpufSxqci5tZWFuPWZ1bmN0aW9uKGUpe3JldHVybiBJdChlLG5jKX0sanIubWVhbkJ5PWZ1bmN0aW9uKGUsdCl7cmV0dXJuIEl0KGUsc28odCwyKSl9LGpyLm1pbj1mdW5jdGlvbihlKXtyZXR1cm4gZSYmZS5sZW5ndGg/X2koZSxuYyxQaSk6bn0sanIubWluQnk9ZnVuY3Rpb24oZSx0KXtyZXR1cm4gZSYmZS5sZW5ndGg/X2koZSxzbyh0LDIpLFBpKTpufSxqci5zdHViQXJyYXk9dmMsanIuc3R1YkZhbHNlPWdjLGpyLnN0dWJPYmplY3Q9ZnVuY3Rpb24oKXtyZXR1cm57fX0sanIuc3R1YlN0cmluZz1mdW5jdGlvbigpe3JldHVybiIifSxqci5zdHViVHJ1ZT1mdW5jdGlvbigpe3JldHVybiEwfSxqci5tdWx0aXBseT13Yyxqci5udGg9ZnVuY3Rpb24oZSx0KXtyZXR1cm4gZSYmZS5sZW5ndGg/V2koZSxwYSh0KSk6bn0sanIubm9Db25mbGljdD1mdW5jdGlvbigpe3JldHVybiBvdC5fPT09dGhpcyYmKG90Ll89amUpLHRoaXN9LGpyLm5vb3A9bGMsanIubm93PUFzLGpyLnBhZD1mdW5jdGlvbihlLHQscil7ZT1tYShlKTt2YXIgaT0odD1wYSh0KSk/bnIoZSk6MDtpZighdHx8aT49dClyZXR1cm4gZTt2YXIgbj0odC1pKS8yO3JldHVybiBxbih1cihuKSxyKStlK3FuKGxyKG4pLHIpfSxqci5wYWRFbmQ9ZnVuY3Rpb24oZSx0LHIpe2U9bWEoZSk7dmFyIGk9KHQ9cGEodCkpP25yKGUpOjA7cmV0dXJuIHQmJmk8dD9lK3FuKHQtaSxyKTplfSxqci5wYWRTdGFydD1mdW5jdGlvbihlLHQscil7ZT1tYShlKTt2YXIgaT0odD1wYSh0KSk/bnIoZSk6MDtyZXR1cm4gdCYmaTx0P3FuKHQtaSxyKStlOmV9LGpyLnBhcnNlSW50PWZ1bmN0aW9uKGUsdCxyKXtyZXR1cm4gcnx8bnVsbD09dD90PTA6dCYmKHQ9K3QpLG1yKG1hKGUpLnJlcGxhY2UoaWUsIiIpLHR8fDApfSxqci5yYW5kb209ZnVuY3Rpb24oZSx0LHIpe2lmKHImJiJib29sZWFuIiE9dHlwZW9mIHImJnlvKGUsdCxyKSYmKHQ9cj1uKSxyPT09biYmKCJib29sZWFuIj09dHlwZW9mIHQ/KHI9dCx0PW4pOiJib29sZWFuIj09dHlwZW9mIGUmJihyPWUsZT1uKSksZT09PW4mJnQ9PT1uPyhlPTAsdD0xKTooZT1kYShlKSx0PT09bj8odD1lLGU9MCk6dD1kYSh0KSksZT50KXt2YXIgaT1lO2U9dCx0PWl9aWYocnx8ZSUxfHx0JTEpe3ZhciBvPWJyKCk7cmV0dXJuIGdyKGUrbyoodC1lK3R0KCIxZS0iKygobysiIikubGVuZ3RoLTEpKSksdCl9cmV0dXJuIEtpKGUsdCl9LGpyLnJlZHVjZT1mdW5jdGlvbihlLHQscil7dmFyIGk9S3MoZSk/QXQ6RnQsbj1hcmd1bWVudHMubGVuZ3RoPDM7cmV0dXJuIGkoZSxzbyh0LDQpLHIsbix1aSl9LGpyLnJlZHVjZVJpZ2h0PWZ1bmN0aW9uKGUsdCxyKXt2YXIgaT1LcyhlKT9rdDpGdCxuPWFyZ3VtZW50cy5sZW5ndGg8MztyZXR1cm4gaShlLHNvKHQsNCkscixuLGhpKX0sanIucmVwZWF0PWZ1bmN0aW9uKGUsdCxyKXtyZXR1cm4gdD0ocj95byhlLHQscik6dD09PW4pPzE6cGEodCksVmkobWEoZSksdCl9LGpyLnJlcGxhY2U9ZnVuY3Rpb24oKXt2YXIgZT1hcmd1bWVudHMsdD1tYShlWzBdKTtyZXR1cm4gZS5sZW5ndGg8Mz90OnQucmVwbGFjZShlWzFdLGVbMl0pfSxqci5yZXN1bHQ9ZnVuY3Rpb24oZSx0LHIpe3ZhciBpPS0xLG89KHQ9Z24odCxlKSkubGVuZ3RoO2ZvcihvfHwobz0xLGU9bik7KytpPG87KXt2YXIgcz1udWxsPT1lP246ZVtqbyh0W2ldKV07cz09PW4mJihpPW8scz1yKSxlPSRzKHMpP3MuY2FsbChlKTpzfXJldHVybiBlfSxqci5yb3VuZD1MYyxqci5ydW5JbkNvbnRleHQ9ZSxqci5zYW1wbGU9ZnVuY3Rpb24oZSl7cmV0dXJuKEtzKGUpP1hyOllpKShlKX0sanIuc2l6ZT1mdW5jdGlvbihlKXtpZihudWxsPT1lKXJldHVybiAwO2lmKEdzKGUpKXJldHVybiBjYShlKT9ucihlKTplLmxlbmd0aDt2YXIgdD1mbyhlKTtyZXR1cm4gdD09Q3x8dD09QT9lLnNpemU6RGkoZSkubGVuZ3RofSxqci5zbmFrZUNhc2U9WWEsanIuc29tZT1mdW5jdGlvbihlLHQscil7dmFyIGk9S3MoZSk/TXQ6dG47cmV0dXJuIHImJnlvKGUsdCxyKSYmKHQ9biksaShlLHNvKHQsMykpfSxqci5zb3J0ZWRJbmRleD1mdW5jdGlvbihlLHQpe3JldHVybiBybihlLHQpfSxqci5zb3J0ZWRJbmRleEJ5PWZ1bmN0aW9uKGUsdCxyKXtyZXR1cm4gbm4oZSx0LHNvKHIsMikpfSxqci5zb3J0ZWRJbmRleE9mPWZ1bmN0aW9uKGUsdCl7dmFyIHI9bnVsbD09ZT8wOmUubGVuZ3RoO2lmKHIpe3ZhciBpPXJuKGUsdCk7aWYoaTxyJiZVcyhlW2ldLHQpKXJldHVybiBpfXJldHVybi0xfSxqci5zb3J0ZWRMYXN0SW5kZXg9ZnVuY3Rpb24oZSx0KXtyZXR1cm4gcm4oZSx0LCEwKX0sanIuc29ydGVkTGFzdEluZGV4Qnk9ZnVuY3Rpb24oZSx0LHIpe3JldHVybiBubihlLHQsc28ociwyKSwhMCl9LGpyLnNvcnRlZExhc3RJbmRleE9mPWZ1bmN0aW9uKGUsdCl7aWYobnVsbCE9ZSYmZS5sZW5ndGgpe3ZhciByPXJuKGUsdCwhMCktMTtpZihVcyhlW3JdLHQpKXJldHVybiByfXJldHVybi0xfSxqci5zdGFydENhc2U9WGEsanIuc3RhcnRzV2l0aD1mdW5jdGlvbihlLHQscil7cmV0dXJuIGU9bWEoZSkscj1udWxsPT1yPzA6b2kocGEociksMCxlLmxlbmd0aCksdD1hbih0KSxlLnNsaWNlKHIscit0Lmxlbmd0aCk9PXR9LGpyLnN1YnRyYWN0PUVjLGpyLnN1bT1mdW5jdGlvbihlKXtyZXR1cm4gZSYmZS5sZW5ndGg/V3QoZSxuYyk6MH0sanIuc3VtQnk9ZnVuY3Rpb24oZSx0KXtyZXR1cm4gZSYmZS5sZW5ndGg/V3QoZSxzbyh0LDIpKTowfSxqci50ZW1wbGF0ZT1mdW5jdGlvbihlLHQscil7dmFyIGk9anIudGVtcGxhdGVTZXR0aW5ncztyJiZ5byhlLHQscikmJih0PW4pLGU9bWEoZSksdD1DYSh7fSx0LGksWm4pO3ZhciBvLHMsYT1DYSh7fSx0LmltcG9ydHMsaS5pbXBvcnRzLFpuKSxjPU9hKGEpLGw9enQoYSxjKSx1PTAsaD10LmludGVycG9sYXRlfHxtZSxmPSJfX3AgKz0gJyIsXz1FZSgodC5lc2NhcGV8fG1lKS5zb3VyY2UrInwiK2guc291cmNlKyJ8IisoaD09PUo/aGU6bWUpLnNvdXJjZSsifCIrKHQuZXZhbHVhdGV8fG1lKS5zb3VyY2UrInwkIiwiZyIpLGQ9Ii8vIyBzb3VyY2VVUkw9IisoQmUuY2FsbCh0LCJzb3VyY2VVUkwiKT8odC5zb3VyY2VVUkwrIiIpLnJlcGxhY2UoL1xzL2csIiAiKToibG9kYXNoLnRlbXBsYXRlU291cmNlc1siKyArK0plKyJdIikrIlxuIjtlLnJlcGxhY2UoXywoZnVuY3Rpb24odCxyLGksbixhLGMpe3JldHVybiBpfHwoaT1uKSxmKz1lLnNsaWNlKHUsYykucmVwbGFjZShiZSxKdCksciYmKG89ITAsZis9IicgK1xuX19lKCIrcisiKSArXG4nIiksYSYmKHM9ITAsZis9Iic7XG4iK2ErIjtcbl9fcCArPSAnIiksaSYmKGYrPSInICtcbigoX190ID0gKCIraSsiKSkgPT0gbnVsbCA/ICcnIDogX190KSArXG4nIiksdT1jK3QubGVuZ3RoLHR9KSksZis9Iic7XG4iO3ZhciBwPUJlLmNhbGwodCwidmFyaWFibGUiKSYmdC52YXJpYWJsZTtpZihwKXtpZihsZS50ZXN0KHApKXRocm93IG5ldyBTZSgiSW52YWxpZCBgdmFyaWFibGVgIG9wdGlvbiBwYXNzZWQgaW50byBgXy50ZW1wbGF0ZWAiKX1lbHNlIGY9IndpdGggKG9iaikge1xuIitmKyJcbn1cbiI7Zj0ocz9mLnJlcGxhY2UocSwiIik6ZikucmVwbGFjZShOLCIkMSIpLnJlcGxhY2UoeiwiJDE7IiksZj0iZnVuY3Rpb24oIisocHx8Im9iaiIpKyIpIHtcbiIrKHA/IiI6Im9iaiB8fCAob2JqID0ge30pO1xuIikrInZhciBfX3QsIF9fcCA9ICcnIisobz8iLCBfX2UgPSBfLmVzY2FwZSI6IiIpKyhzPyIsIF9faiA9IEFycmF5LnByb3RvdHlwZS5qb2luO1xuZnVuY3Rpb24gcHJpbnQoKSB7IF9fcCArPSBfX2ouY2FsbChhcmd1bWVudHMsICcnKSB9XG4iOiI7XG4iKStmKyJyZXR1cm4gX19wXG59Ijt2YXIgdj1RYSgoZnVuY3Rpb24oKXtyZXR1cm4gQ2UoYyxkKyJyZXR1cm4gIitmKS5hcHBseShuLGwpfSkpO2lmKHYuc291cmNlPWYsSnModikpdGhyb3cgdjtyZXR1cm4gdn0sanIudGltZXM9ZnVuY3Rpb24oZSx0KXtpZigoZT1wYShlKSk8MXx8ZT5oKXJldHVybltdO3ZhciByPV8saT1ncihlLF8pO3Q9c28odCksZS09Xztmb3IodmFyIG49VXQoaSx0KTsrK3I8ZTspdChyKTtyZXR1cm4gbn0sanIudG9GaW5pdGU9ZGEsanIudG9JbnRlZ2VyPXBhLGpyLnRvTGVuZ3RoPXZhLGpyLnRvTG93ZXI9ZnVuY3Rpb24oZSl7cmV0dXJuIG1hKGUpLnRvTG93ZXJDYXNlKCl9LGpyLnRvTnVtYmVyPWdhLGpyLnRvU2FmZUludGVnZXI9ZnVuY3Rpb24oZSl7cmV0dXJuIGU/b2kocGEoZSksLTkwMDcxOTkyNTQ3NDA5OTEsaCk6MD09PWU/ZTowfSxqci50b1N0cmluZz1tYSxqci50b1VwcGVyPWZ1bmN0aW9uKGUpe3JldHVybiBtYShlKS50b1VwcGVyQ2FzZSgpfSxqci50cmltPWZ1bmN0aW9uKGUsdCxyKXtpZigoZT1tYShlKSkmJihyfHx0PT09bikpcmV0dXJuIHF0KGUpO2lmKCFlfHwhKHQ9YW4odCkpKXJldHVybiBlO3ZhciBpPW9yKGUpLG89b3IodCk7cmV0dXJuIG1uKGksVnQoaSxvKSxHdChpLG8pKzEpLmpvaW4oIiIpfSxqci50cmltRW5kPWZ1bmN0aW9uKGUsdCxyKXtpZigoZT1tYShlKSkmJihyfHx0PT09bikpcmV0dXJuIGUuc2xpY2UoMCxzcihlKSsxKTtpZighZXx8ISh0PWFuKHQpKSlyZXR1cm4gZTt2YXIgaT1vcihlKTtyZXR1cm4gbW4oaSwwLEd0KGksb3IodCkpKzEpLmpvaW4oIiIpfSxqci50cmltU3RhcnQ9ZnVuY3Rpb24oZSx0LHIpe2lmKChlPW1hKGUpKSYmKHJ8fHQ9PT1uKSlyZXR1cm4gZS5yZXBsYWNlKGllLCIiKTtpZighZXx8ISh0PWFuKHQpKSlyZXR1cm4gZTt2YXIgaT1vcihlKTtyZXR1cm4gbW4oaSxWdChpLG9yKHQpKSkuam9pbigiIil9LGpyLnRydW5jYXRlPWZ1bmN0aW9uKGUsdCl7dmFyIHI9MzAsaT0iLi4uIjtpZih0YSh0KSl7dmFyIG89InNlcGFyYXRvciJpbiB0P3Quc2VwYXJhdG9yOm87cj0ibGVuZ3RoImluIHQ/cGEodC5sZW5ndGgpOnIsaT0ib21pc3Npb24iaW4gdD9hbih0Lm9taXNzaW9uKTppfXZhciBzPShlPW1hKGUpKS5sZW5ndGg7aWYoJHQoZSkpe3ZhciBhPW9yKGUpO3M9YS5sZW5ndGh9aWYocj49cylyZXR1cm4gZTt2YXIgYz1yLW5yKGkpO2lmKGM8MSlyZXR1cm4gaTt2YXIgbD1hP21uKGEsMCxjKS5qb2luKCIiKTplLnNsaWNlKDAsYyk7aWYobz09PW4pcmV0dXJuIGwraTtpZihhJiYoYys9bC5sZW5ndGgtYyksc2Eobykpe2lmKGUuc2xpY2UoYykuc2VhcmNoKG8pKXt2YXIgdSxoPWw7Zm9yKG8uZ2xvYmFsfHwobz1FZShvLnNvdXJjZSxtYShmZS5leGVjKG8pKSsiZyIpKSxvLmxhc3RJbmRleD0wO3U9by5leGVjKGgpOyl2YXIgZj11LmluZGV4O2w9bC5zbGljZSgwLGY9PT1uP2M6Zil9fWVsc2UgaWYoZS5pbmRleE9mKGFuKG8pLGMpIT1jKXt2YXIgXz1sLmxhc3RJbmRleE9mKG8pO18+LTEmJihsPWwuc2xpY2UoMCxfKSl9cmV0dXJuIGwraX0sanIudW5lc2NhcGU9ZnVuY3Rpb24oZSl7cmV0dXJuKGU9bWEoZSkpJiZHLnRlc3QoZSk/ZS5yZXBsYWNlKEssYXIpOmV9LGpyLnVuaXF1ZUlkPWZ1bmN0aW9uKGUpe3ZhciB0PSsrRGU7cmV0dXJuIG1hKGUpK3R9LGpyLnVwcGVyQ2FzZT1aYSxqci51cHBlckZpcnN0PUphLGpyLmVhY2g9bXMsanIuZWFjaFJpZ2h0PWJzLGpyLmZpcnN0PUdvLGNjKGpyLCh5Yz17fSx5aShqciwoZnVuY3Rpb24oZSx0KXtCZS5jYWxsKGpyLnByb3RvdHlwZSx0KXx8KHljW3RdPWUpfSkpLHljKSx7Y2hhaW46ITF9KSxqci5WRVJTSU9OPSI0LjE3LjIxIixtdChbImJpbmQiLCJiaW5kS2V5IiwiY3VycnkiLCJjdXJyeVJpZ2h0IiwicGFydGlhbCIsInBhcnRpYWxSaWdodCJdLChmdW5jdGlvbihlKXtqcltlXS5wbGFjZWhvbGRlcj1qcn0pKSxtdChbImRyb3AiLCJ0YWtlIl0sKGZ1bmN0aW9uKGUsdCl7cXIucHJvdG90eXBlW2VdPWZ1bmN0aW9uKHIpe3I9cj09PW4/MTp2cihwYShyKSwwKTt2YXIgaT10aGlzLl9fZmlsdGVyZWRfXyYmIXQ/bmV3IHFyKHRoaXMpOnRoaXMuY2xvbmUoKTtyZXR1cm4gaS5fX2ZpbHRlcmVkX18/aS5fX3Rha2VDb3VudF9fPWdyKHIsaS5fX3Rha2VDb3VudF9fKTppLl9fdmlld3NfXy5wdXNoKHtzaXplOmdyKHIsXyksdHlwZTplKyhpLl9fZGlyX188MD8iUmlnaHQiOiIiKX0pLGl9LHFyLnByb3RvdHlwZVtlKyJSaWdodCJdPWZ1bmN0aW9uKHQpe3JldHVybiB0aGlzLnJldmVyc2UoKVtlXSh0KS5yZXZlcnNlKCl9fSkpLG10KFsiZmlsdGVyIiwibWFwIiwidGFrZVdoaWxlIl0sKGZ1bmN0aW9uKGUsdCl7dmFyIHI9dCsxLGk9MT09cnx8Mz09cjtxci5wcm90b3R5cGVbZV09ZnVuY3Rpb24oZSl7dmFyIHQ9dGhpcy5jbG9uZSgpO3JldHVybiB0Ll9faXRlcmF0ZWVzX18ucHVzaCh7aXRlcmF0ZWU6c28oZSwzKSx0eXBlOnJ9KSx0Ll9fZmlsdGVyZWRfXz10Ll9fZmlsdGVyZWRfX3x8aSx0fX0pKSxtdChbImhlYWQiLCJsYXN0Il0sKGZ1bmN0aW9uKGUsdCl7dmFyIHI9InRha2UiKyh0PyJSaWdodCI6IiIpO3FyLnByb3RvdHlwZVtlXT1mdW5jdGlvbigpe3JldHVybiB0aGlzW3JdKDEpLnZhbHVlKClbMF19fSkpLG10KFsiaW5pdGlhbCIsInRhaWwiXSwoZnVuY3Rpb24oZSx0KXt2YXIgcj0iZHJvcCIrKHQ/IiI6IlJpZ2h0Iik7cXIucHJvdG90eXBlW2VdPWZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX19maWx0ZXJlZF9fP25ldyBxcih0aGlzKTp0aGlzW3JdKDEpfX0pKSxxci5wcm90b3R5cGUuY29tcGFjdD1mdW5jdGlvbigpe3JldHVybiB0aGlzLmZpbHRlcihuYyl9LHFyLnByb3RvdHlwZS5maW5kPWZ1bmN0aW9uKGUpe3JldHVybiB0aGlzLmZpbHRlcihlKS5oZWFkKCl9LHFyLnByb3RvdHlwZS5maW5kTGFzdD1mdW5jdGlvbihlKXtyZXR1cm4gdGhpcy5yZXZlcnNlKCkuZmluZChlKX0scXIucHJvdG90eXBlLmludm9rZU1hcD1HaSgoZnVuY3Rpb24oZSx0KXtyZXR1cm4iZnVuY3Rpb24iPT10eXBlb2YgZT9uZXcgcXIodGhpcyk6dGhpcy5tYXAoKGZ1bmN0aW9uKHIpe3JldHVybiBraShyLGUsdCl9KSl9KSkscXIucHJvdG90eXBlLnJlamVjdD1mdW5jdGlvbihlKXtyZXR1cm4gdGhpcy5maWx0ZXIoSXMoc28oZSkpKX0scXIucHJvdG90eXBlLnNsaWNlPWZ1bmN0aW9uKGUsdCl7ZT1wYShlKTt2YXIgcj10aGlzO3JldHVybiByLl9fZmlsdGVyZWRfXyYmKGU+MHx8dDwwKT9uZXcgcXIocik6KGU8MD9yPXIudGFrZVJpZ2h0KC1lKTplJiYocj1yLmRyb3AoZSkpLHQhPT1uJiYocj0odD1wYSh0KSk8MD9yLmRyb3BSaWdodCgtdCk6ci50YWtlKHQtZSkpLHIpfSxxci5wcm90b3R5cGUudGFrZVJpZ2h0V2hpbGU9ZnVuY3Rpb24oZSl7cmV0dXJuIHRoaXMucmV2ZXJzZSgpLnRha2VXaGlsZShlKS5yZXZlcnNlKCl9LHFyLnByb3RvdHlwZS50b0FycmF5PWZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMudGFrZShfKX0seWkocXIucHJvdG90eXBlLChmdW5jdGlvbihlLHQpe3ZhciByPS9eKD86ZmlsdGVyfGZpbmR8bWFwfHJlamVjdCl8V2hpbGUkLy50ZXN0KHQpLGk9L14oPzpoZWFkfGxhc3QpJC8udGVzdCh0KSxvPWpyW2k/InRha2UiKygibGFzdCI9PXQ/IlJpZ2h0IjoiIik6dF0scz1pfHwvXmZpbmQvLnRlc3QodCk7byYmKGpyLnByb3RvdHlwZVt0XT1mdW5jdGlvbigpe3ZhciB0PXRoaXMuX193cmFwcGVkX18sYT1pP1sxXTphcmd1bWVudHMsYz10IGluc3RhbmNlb2YgcXIsbD1hWzBdLHU9Y3x8S3ModCksaD1mdW5jdGlvbihlKXt2YXIgdD1vLmFwcGx5KGpyLHh0KFtlXSxhKSk7cmV0dXJuIGkmJmY/dFswXTp0fTt1JiZyJiYiZnVuY3Rpb24iPT10eXBlb2YgbCYmMSE9bC5sZW5ndGgmJihjPXU9ITEpO3ZhciBmPXRoaXMuX19jaGFpbl9fLF89ISF0aGlzLl9fYWN0aW9uc19fLmxlbmd0aCxkPXMmJiFmLHA9YyYmIV87aWYoIXMmJnUpe3Q9cD90Om5ldyBxcih0aGlzKTt2YXIgdj1lLmFwcGx5KHQsYSk7cmV0dXJuIHYuX19hY3Rpb25zX18ucHVzaCh7ZnVuYzpkcyxhcmdzOltoXSx0aGlzQXJnOm59KSxuZXcgVXIodixmKX1yZXR1cm4gZCYmcD9lLmFwcGx5KHRoaXMsYSk6KHY9dGhpcy50aHJ1KGgpLGQ/aT92LnZhbHVlKClbMF06di52YWx1ZSgpOnYpfSl9KSksbXQoWyJwb3AiLCJwdXNoIiwic2hpZnQiLCJzb3J0Iiwic3BsaWNlIiwidW5zaGlmdCJdLChmdW5jdGlvbihlKXt2YXIgdD1rZVtlXSxyPS9eKD86cHVzaHxzb3J0fHVuc2hpZnQpJC8udGVzdChlKT8idGFwIjoidGhydSIsaT0vXig/OnBvcHxzaGlmdCkkLy50ZXN0KGUpO2pyLnByb3RvdHlwZVtlXT1mdW5jdGlvbigpe3ZhciBlPWFyZ3VtZW50cztpZihpJiYhdGhpcy5fX2NoYWluX18pe3ZhciBuPXRoaXMudmFsdWUoKTtyZXR1cm4gdC5hcHBseShLcyhuKT9uOltdLGUpfXJldHVybiB0aGlzW3JdKChmdW5jdGlvbihyKXtyZXR1cm4gdC5hcHBseShLcyhyKT9yOltdLGUpfSkpfX0pKSx5aShxci5wcm90b3R5cGUsKGZ1bmN0aW9uKGUsdCl7dmFyIHI9anJbdF07aWYocil7dmFyIGk9ci5uYW1lKyIiO0JlLmNhbGwoTXIsaSl8fChNcltpXT1bXSksTXJbaV0ucHVzaCh7bmFtZTp0LGZ1bmM6cn0pfX0pKSxNcltqbihuLDIpLm5hbWVdPVt7bmFtZToid3JhcHBlciIsZnVuYzpufV0scXIucHJvdG90eXBlLmNsb25lPWZ1bmN0aW9uKCl7dmFyIGU9bmV3IHFyKHRoaXMuX193cmFwcGVkX18pO3JldHVybiBlLl9fYWN0aW9uc19fPUFuKHRoaXMuX19hY3Rpb25zX18pLGUuX19kaXJfXz10aGlzLl9fZGlyX18sZS5fX2ZpbHRlcmVkX189dGhpcy5fX2ZpbHRlcmVkX18sZS5fX2l0ZXJhdGVlc19fPUFuKHRoaXMuX19pdGVyYXRlZXNfXyksZS5fX3Rha2VDb3VudF9fPXRoaXMuX190YWtlQ291bnRfXyxlLl9fdmlld3NfXz1Bbih0aGlzLl9fdmlld3NfXyksZX0scXIucHJvdG90eXBlLnJldmVyc2U9ZnVuY3Rpb24oKXtpZih0aGlzLl9fZmlsdGVyZWRfXyl7dmFyIGU9bmV3IHFyKHRoaXMpO2UuX19kaXJfXz0tMSxlLl9fZmlsdGVyZWRfXz0hMH1lbHNlKGU9dGhpcy5jbG9uZSgpKS5fX2Rpcl9fKj0tMTtyZXR1cm4gZX0scXIucHJvdG90eXBlLnZhbHVlPWZ1bmN0aW9uKCl7dmFyIGU9dGhpcy5fX3dyYXBwZWRfXy52YWx1ZSgpLHQ9dGhpcy5fX2Rpcl9fLHI9S3MoZSksaT10PDAsbj1yP2UubGVuZ3RoOjAsbz1mdW5jdGlvbihlLHQscil7Zm9yKHZhciBpPS0xLG49ci5sZW5ndGg7KytpPG47KXt2YXIgbz1yW2ldLHM9by5zaXplO3N3aXRjaChvLnR5cGUpe2Nhc2UiZHJvcCI6ZSs9czticmVhaztjYXNlImRyb3BSaWdodCI6dC09czticmVhaztjYXNlInRha2UiOnQ9Z3IodCxlK3MpO2JyZWFrO2Nhc2UidGFrZVJpZ2h0IjplPXZyKGUsdC1zKX19cmV0dXJue3N0YXJ0OmUsZW5kOnR9fSgwLG4sdGhpcy5fX3ZpZXdzX18pLHM9by5zdGFydCxhPW8uZW5kLGM9YS1zLGw9aT9hOnMtMSx1PXRoaXMuX19pdGVyYXRlZXNfXyxoPXUubGVuZ3RoLGY9MCxfPWdyKGMsdGhpcy5fX3Rha2VDb3VudF9fKTtpZighcnx8IWkmJm49PWMmJl89PWMpcmV0dXJuIGZuKGUsdGhpcy5fX2FjdGlvbnNfXyk7dmFyIGQ9W107ZTpmb3IoO2MtLSYmZjxfOyl7Zm9yKHZhciBwPS0xLHY9ZVtsKz10XTsrK3A8aDspe3ZhciBnPXVbcF0seT1nLml0ZXJhdGVlLG09Zy50eXBlLGI9eSh2KTtpZigyPT1tKXY9YjtlbHNlIGlmKCFiKXtpZigxPT1tKWNvbnRpbnVlIGU7YnJlYWsgZX19ZFtmKytdPXZ9cmV0dXJuIGR9LGpyLnByb3RvdHlwZS5hdD1wcyxqci5wcm90b3R5cGUuY2hhaW49ZnVuY3Rpb24oKXtyZXR1cm4gX3ModGhpcyl9LGpyLnByb3RvdHlwZS5jb21taXQ9ZnVuY3Rpb24oKXtyZXR1cm4gbmV3IFVyKHRoaXMudmFsdWUoKSx0aGlzLl9fY2hhaW5fXyl9LGpyLnByb3RvdHlwZS5uZXh0PWZ1bmN0aW9uKCl7dGhpcy5fX3ZhbHVlc19fPT09biYmKHRoaXMuX192YWx1ZXNfXz1fYSh0aGlzLnZhbHVlKCkpKTt2YXIgZT10aGlzLl9faW5kZXhfXz49dGhpcy5fX3ZhbHVlc19fLmxlbmd0aDtyZXR1cm57ZG9uZTplLHZhbHVlOmU/bjp0aGlzLl9fdmFsdWVzX19bdGhpcy5fX2luZGV4X18rK119fSxqci5wcm90b3R5cGUucGxhbnQ9ZnVuY3Rpb24oZSl7Zm9yKHZhciB0LHI9dGhpcztyIGluc3RhbmNlb2YgV3I7KXt2YXIgaT1XbyhyKTtpLl9faW5kZXhfXz0wLGkuX192YWx1ZXNfXz1uLHQ/by5fX3dyYXBwZWRfXz1pOnQ9aTt2YXIgbz1pO3I9ci5fX3dyYXBwZWRfX31yZXR1cm4gby5fX3dyYXBwZWRfXz1lLHR9LGpyLnByb3RvdHlwZS5yZXZlcnNlPWZ1bmN0aW9uKCl7dmFyIGU9dGhpcy5fX3dyYXBwZWRfXztpZihlIGluc3RhbmNlb2YgcXIpe3ZhciB0PWU7cmV0dXJuIHRoaXMuX19hY3Rpb25zX18ubGVuZ3RoJiYodD1uZXcgcXIodGhpcykpLCh0PXQucmV2ZXJzZSgpKS5fX2FjdGlvbnNfXy5wdXNoKHtmdW5jOmRzLGFyZ3M6W3RzXSx0aGlzQXJnOm59KSxuZXcgVXIodCx0aGlzLl9fY2hhaW5fXyl9cmV0dXJuIHRoaXMudGhydSh0cyl9LGpyLnByb3RvdHlwZS50b0pTT049anIucHJvdG90eXBlLnZhbHVlT2Y9anIucHJvdG90eXBlLnZhbHVlPWZ1bmN0aW9uKCl7cmV0dXJuIGZuKHRoaXMuX193cmFwcGVkX18sdGhpcy5fX2FjdGlvbnNfXyl9LGpyLnByb3RvdHlwZS5maXJzdD1qci5wcm90b3R5cGUuaGVhZCxzdCYmKGpyLnByb3RvdHlwZVtzdF09ZnVuY3Rpb24oKXtyZXR1cm4gdGhpc30pLGpyfSgpO290Ll89Y3IsKGk9ZnVuY3Rpb24oKXtyZXR1cm4gY3J9LmNhbGwodCxyLHQsZSkpPT09bnx8KGUuZXhwb3J0cz1pKX0uY2FsbCh0aGlzKX0sMzc5OmU9PnsidXNlIHN0cmljdCI7dmFyIHQ9W107ZnVuY3Rpb24gcihlKXtmb3IodmFyIHI9LTEsaT0wO2k8dC5sZW5ndGg7aSsrKWlmKHRbaV0uaWRlbnRpZmllcj09PWUpe3I9aTticmVha31yZXR1cm4gcn1mdW5jdGlvbiBpKGUsaSl7Zm9yKHZhciBvPXt9LHM9W10sYT0wO2E8ZS5sZW5ndGg7YSsrKXt2YXIgYz1lW2FdLGw9aS5iYXNlP2NbMF0raS5iYXNlOmNbMF0sdT1vW2xdfHwwLGg9IiIuY29uY2F0KGwsIiAiKS5jb25jYXQodSk7b1tsXT11KzE7dmFyIGY9cihoKSxfPXtjc3M6Y1sxXSxtZWRpYTpjWzJdLHNvdXJjZU1hcDpjWzNdLHN1cHBvcnRzOmNbNF0sbGF5ZXI6Y1s1XX07aWYoLTEhPT1mKXRbZl0ucmVmZXJlbmNlcysrLHRbZl0udXBkYXRlcihfKTtlbHNle3ZhciBkPW4oXyxpKTtpLmJ5SW5kZXg9YSx0LnNwbGljZShhLDAse2lkZW50aWZpZXI6aCx1cGRhdGVyOmQscmVmZXJlbmNlczoxfSl9cy5wdXNoKGgpfXJldHVybiBzfWZ1bmN0aW9uIG4oZSx0KXt2YXIgcj10LmRvbUFQSSh0KTtyZXR1cm4gci51cGRhdGUoZSksZnVuY3Rpb24odCl7aWYodCl7aWYodC5jc3M9PT1lLmNzcyYmdC5tZWRpYT09PWUubWVkaWEmJnQuc291cmNlTWFwPT09ZS5zb3VyY2VNYXAmJnQuc3VwcG9ydHM9PT1lLnN1cHBvcnRzJiZ0LmxheWVyPT09ZS5sYXllcilyZXR1cm47ci51cGRhdGUoZT10KX1lbHNlIHIucmVtb3ZlKCl9fWUuZXhwb3J0cz1mdW5jdGlvbihlLG4pe3ZhciBvPWkoZT1lfHxbXSxuPW58fHt9KTtyZXR1cm4gZnVuY3Rpb24oZSl7ZT1lfHxbXTtmb3IodmFyIHM9MDtzPG8ubGVuZ3RoO3MrKyl7dmFyIGE9cihvW3NdKTt0W2FdLnJlZmVyZW5jZXMtLX1mb3IodmFyIGM9aShlLG4pLGw9MDtsPG8ubGVuZ3RoO2wrKyl7dmFyIHU9cihvW2xdKTswPT09dFt1XS5yZWZlcmVuY2VzJiYodFt1XS51cGRhdGVyKCksdC5zcGxpY2UodSwxKSl9bz1jfX19LDU2OTplPT57InVzZSBzdHJpY3QiO3ZhciB0PXt9O2UuZXhwb3J0cz1mdW5jdGlvbihlLHIpe3ZhciBpPWZ1bmN0aW9uKGUpe2lmKHZvaWQgMD09PXRbZV0pe3ZhciByPWRvY3VtZW50LnF1ZXJ5U2VsZWN0b3IoZSk7aWYod2luZG93LkhUTUxJRnJhbWVFbGVtZW50JiZyIGluc3RhbmNlb2Ygd2luZG93LkhUTUxJRnJhbWVFbGVtZW50KXRyeXtyPXIuY29udGVudERvY3VtZW50LmhlYWR9Y2F0Y2goZSl7cj1udWxsfXRbZV09cn1yZXR1cm4gdFtlXX0oZSk7aWYoIWkpdGhyb3cgbmV3IEVycm9yKCJDb3VsZG4ndCBmaW5kIGEgc3R5bGUgdGFyZ2V0LiBUaGlzIHByb2JhYmx5IG1lYW5zIHRoYXQgdGhlIHZhbHVlIGZvciB0aGUgJ2luc2VydCcgcGFyYW1ldGVyIGlzIGludmFsaWQuIik7aS5hcHBlbmRDaGlsZChyKX19LDIxNjplPT57InVzZSBzdHJpY3QiO2UuZXhwb3J0cz1mdW5jdGlvbihlKXt2YXIgdD1kb2N1bWVudC5jcmVhdGVFbGVtZW50KCJzdHlsZSIpO3JldHVybiBlLnNldEF0dHJpYnV0ZXModCxlLmF0dHJpYnV0ZXMpLGUuaW5zZXJ0KHQsZS5vcHRpb25zKSx0fX0sNTY1OihlLHQscik9PnsidXNlIHN0cmljdCI7ZS5leHBvcnRzPWZ1bmN0aW9uKGUpe3ZhciB0PXIubmM7dCYmZS5zZXRBdHRyaWJ1dGUoIm5vbmNlIix0KX19LDc5NTplPT57InVzZSBzdHJpY3QiO2UuZXhwb3J0cz1mdW5jdGlvbihlKXt2YXIgdD1lLmluc2VydFN0eWxlRWxlbWVudChlKTtyZXR1cm57dXBkYXRlOmZ1bmN0aW9uKHIpeyFmdW5jdGlvbihlLHQscil7dmFyIGk9IiI7ci5zdXBwb3J0cyYmKGkrPSJAc3VwcG9ydHMgKCIuY29uY2F0KHIuc3VwcG9ydHMsIikgeyIpKSxyLm1lZGlhJiYoaSs9IkBtZWRpYSAiLmNvbmNhdChyLm1lZGlhLCIgeyIpKTt2YXIgbj12b2lkIDAhPT1yLmxheWVyO24mJihpKz0iQGxheWVyIi5jb25jYXQoci5sYXllci5sZW5ndGg+MD8iICIuY29uY2F0KHIubGF5ZXIpOiIiLCIgeyIpKSxpKz1yLmNzcyxuJiYoaSs9In0iKSxyLm1lZGlhJiYoaSs9In0iKSxyLnN1cHBvcnRzJiYoaSs9In0iKTt2YXIgbz1yLnNvdXJjZU1hcDtvJiYidW5kZWZpbmVkIiE9dHlwZW9mIGJ0b2EmJihpKz0iXG4vKiMgc291cmNlTWFwcGluZ1VSTD1kYXRhOmFwcGxpY2F0aW9uL2pzb247YmFzZTY0LCIuY29uY2F0KGJ0b2EodW5lc2NhcGUoZW5jb2RlVVJJQ29tcG9uZW50KEpTT04uc3RyaW5naWZ5KG8pKSkpLCIgKi8iKSksdC5zdHlsZVRhZ1RyYW5zZm9ybShpLGUsdC5vcHRpb25zKX0odCxlLHIpfSxyZW1vdmU6ZnVuY3Rpb24oKXshZnVuY3Rpb24oZSl7aWYobnVsbD09PWUucGFyZW50Tm9kZSlyZXR1cm4hMTtlLnBhcmVudE5vZGUucmVtb3ZlQ2hpbGQoZSl9KHQpfX19fSw1ODk6ZT0+eyJ1c2Ugc3RyaWN0IjtlLmV4cG9ydHM9ZnVuY3Rpb24oZSx0KXtpZih0LnN0eWxlU2hlZXQpdC5zdHlsZVNoZWV0LmNzc1RleHQ9ZTtlbHNle2Zvcig7dC5maXJzdENoaWxkOyl0LnJlbW92ZUNoaWxkKHQuZmlyc3RDaGlsZCk7dC5hcHBlbmRDaGlsZChkb2N1bWVudC5jcmVhdGVUZXh0Tm9kZShlKSl9fX0sNjE3OmU9PntzZWxmLGUuZXhwb3J0cz0oKCk9PnsidXNlIHN0cmljdCI7dmFyIGU9ezc3NTooZSx0KT0+e09iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LCJfX2VzTW9kdWxlIix7dmFsdWU6ITB9KSx0LkZpdEFkZG9uPXZvaWQgMDt2YXIgcj1mdW5jdGlvbigpe2Z1bmN0aW9uIGUoKXt9cmV0dXJuIGUucHJvdG90eXBlLmFjdGl2YXRlPWZ1bmN0aW9uKGUpe3RoaXMuX3Rlcm1pbmFsPWV9LGUucHJvdG90eXBlLmRpc3Bvc2U9ZnVuY3Rpb24oKXt9LGUucHJvdG90eXBlLmZpdD1mdW5jdGlvbigpe3ZhciBlPXRoaXMucHJvcG9zZURpbWVuc2lvbnMoKTtpZihlJiZ0aGlzLl90ZXJtaW5hbCl7dmFyIHQ9dGhpcy5fdGVybWluYWwuX2NvcmU7dGhpcy5fdGVybWluYWwucm93cz09PWUucm93cyYmdGhpcy5fdGVybWluYWwuY29scz09PWUuY29sc3x8KHQuX3JlbmRlclNlcnZpY2UuY2xlYXIoKSx0aGlzLl90ZXJtaW5hbC5yZXNpemUoZS5jb2xzLGUucm93cykpfX0sZS5wcm90b3R5cGUucHJvcG9zZURpbWVuc2lvbnM9ZnVuY3Rpb24oKXtpZih0aGlzLl90ZXJtaW5hbCYmdGhpcy5fdGVybWluYWwuZWxlbWVudCYmdGhpcy5fdGVybWluYWwuZWxlbWVudC5wYXJlbnRFbGVtZW50KXt2YXIgZT10aGlzLl90ZXJtaW5hbC5fY29yZTtpZigwIT09ZS5fcmVuZGVyU2VydmljZS5kaW1lbnNpb25zLmFjdHVhbENlbGxXaWR0aCYmMCE9PWUuX3JlbmRlclNlcnZpY2UuZGltZW5zaW9ucy5hY3R1YWxDZWxsSGVpZ2h0KXt2YXIgdD13aW5kb3cuZ2V0Q29tcHV0ZWRTdHlsZSh0aGlzLl90ZXJtaW5hbC5lbGVtZW50LnBhcmVudEVsZW1lbnQpLHI9cGFyc2VJbnQodC5nZXRQcm9wZXJ0eVZhbHVlKCJoZWlnaHQiKSksaT1NYXRoLm1heCgwLHBhcnNlSW50KHQuZ2V0UHJvcGVydHlWYWx1ZSgid2lkdGgiKSkpLG49d2luZG93LmdldENvbXB1dGVkU3R5bGUodGhpcy5fdGVybWluYWwuZWxlbWVudCksbz1yLShwYXJzZUludChuLmdldFByb3BlcnR5VmFsdWUoInBhZGRpbmctdG9wIikpK3BhcnNlSW50KG4uZ2V0UHJvcGVydHlWYWx1ZSgicGFkZGluZy1ib3R0b20iKSkpLHM9aS0ocGFyc2VJbnQobi5nZXRQcm9wZXJ0eVZhbHVlKCJwYWRkaW5nLXJpZ2h0IikpK3BhcnNlSW50KG4uZ2V0UHJvcGVydHlWYWx1ZSgicGFkZGluZy1sZWZ0IikpKS1lLnZpZXdwb3J0LnNjcm9sbEJhcldpZHRoO3JldHVybntjb2xzOk1hdGgubWF4KDIsTWF0aC5mbG9vcihzL2UuX3JlbmRlclNlcnZpY2UuZGltZW5zaW9ucy5hY3R1YWxDZWxsV2lkdGgpKSxyb3dzOk1hdGgubWF4KDEsTWF0aC5mbG9vcihvL2UuX3JlbmRlclNlcnZpY2UuZGltZW5zaW9ucy5hY3R1YWxDZWxsSGVpZ2h0KSl9fX19LGV9KCk7dC5GaXRBZGRvbj1yfX0sdD17fTtyZXR1cm4gZnVuY3Rpb24gcihpKXtpZih0W2ldKXJldHVybiB0W2ldLmV4cG9ydHM7dmFyIG49dFtpXT17ZXhwb3J0czp7fX07cmV0dXJuIGVbaV0obixuLmV4cG9ydHMsciksbi5leHBvcnRzfSg3NzUpfSkoKX0sMzIwOmU9PntzZWxmLGUuZXhwb3J0cz0oKCk9PnsidXNlIHN0cmljdCI7dmFyIGU9ezQ1Njc6ZnVuY3Rpb24oZSx0LHIpe3ZhciBpLG49dGhpcyYmdGhpcy5fX2V4dGVuZHN8fChpPWZ1bmN0aW9uKGUsdCl7cmV0dXJuIGk9T2JqZWN0LnNldFByb3RvdHlwZU9mfHx7X19wcm90b19fOltdfWluc3RhbmNlb2YgQXJyYXkmJmZ1bmN0aW9uKGUsdCl7ZS5fX3Byb3RvX189dH18fGZ1bmN0aW9uKGUsdCl7Zm9yKHZhciByIGluIHQpT2JqZWN0LnByb3RvdHlwZS5oYXNPd25Qcm9wZXJ0eS5jYWxsKHQscikmJihlW3JdPXRbcl0pfSxpKGUsdCl9LGZ1bmN0aW9uKGUsdCl7aWYoImZ1bmN0aW9uIiE9dHlwZW9mIHQmJm51bGwhPT10KXRocm93IG5ldyBUeXBlRXJyb3IoIkNsYXNzIGV4dGVuZHMgdmFsdWUgIitTdHJpbmcodCkrIiBpcyBub3QgYSBjb25zdHJ1Y3RvciBvciBudWxsIik7ZnVuY3Rpb24gcigpe3RoaXMuY29uc3RydWN0b3I9ZX1pKGUsdCksZS5wcm90b3R5cGU9bnVsbD09PXQ/T2JqZWN0LmNyZWF0ZSh0KTooci5wcm90b3R5cGU9dC5wcm90b3R5cGUsbmV3IHIpfSk7T2JqZWN0LmRlZmluZVByb3BlcnR5KHQsIl9fZXNNb2R1bGUiLHt2YWx1ZTohMH0pLHQuQWNjZXNzaWJpbGl0eU1hbmFnZXI9dm9pZCAwO3ZhciBvPXIoOTA0Mikscz1yKDYxMTQpLGE9cig5OTI0KSxjPXIoMzY1NiksbD1yKDg0NCksdT1yKDU1OTYpLGg9cig5NjMxKSxmPWZ1bmN0aW9uKGUpe2Z1bmN0aW9uIHQodCxyKXt2YXIgaT1lLmNhbGwodGhpcyl8fHRoaXM7aS5fdGVybWluYWw9dCxpLl9yZW5kZXJTZXJ2aWNlPXIsaS5fbGl2ZVJlZ2lvbkxpbmVDb3VudD0wLGkuX2NoYXJzVG9Db25zdW1lPVtdLGkuX2NoYXJzVG9Bbm5vdW5jZT0iIixpLl9hY2Nlc3NpYmlsaXR5VHJlZVJvb3Q9ZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgiZGl2IiksaS5fYWNjZXNzaWJpbGl0eVRyZWVSb290LnNldEF0dHJpYnV0ZSgicm9sZSIsImRvY3VtZW50IiksaS5fYWNjZXNzaWJpbGl0eVRyZWVSb290LmNsYXNzTGlzdC5hZGQoInh0ZXJtLWFjY2Vzc2liaWxpdHkiKSxpLl9hY2Nlc3NpYmlsaXR5VHJlZVJvb3QudGFiSW5kZXg9MCxpLl9yb3dDb250YWluZXI9ZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgiZGl2IiksaS5fcm93Q29udGFpbmVyLnNldEF0dHJpYnV0ZSgicm9sZSIsImxpc3QiKSxpLl9yb3dDb250YWluZXIuY2xhc3NMaXN0LmFkZCgieHRlcm0tYWNjZXNzaWJpbGl0eS10cmVlIiksaS5fcm93RWxlbWVudHM9W107Zm9yKHZhciBuPTA7bjxpLl90ZXJtaW5hbC5yb3dzO24rKylpLl9yb3dFbGVtZW50c1tuXT1pLl9jcmVhdGVBY2Nlc3NpYmlsaXR5VHJlZU5vZGUoKSxpLl9yb3dDb250YWluZXIuYXBwZW5kQ2hpbGQoaS5fcm93RWxlbWVudHNbbl0pO2lmKGkuX3RvcEJvdW5kYXJ5Rm9jdXNMaXN0ZW5lcj1mdW5jdGlvbihlKXtyZXR1cm4gaS5fb25Cb3VuZGFyeUZvY3VzKGUsMCl9LGkuX2JvdHRvbUJvdW5kYXJ5Rm9jdXNMaXN0ZW5lcj1mdW5jdGlvbihlKXtyZXR1cm4gaS5fb25Cb3VuZGFyeUZvY3VzKGUsMSl9LGkuX3Jvd0VsZW1lbnRzWzBdLmFkZEV2ZW50TGlzdGVuZXIoImZvY3VzIixpLl90b3BCb3VuZGFyeUZvY3VzTGlzdGVuZXIpLGkuX3Jvd0VsZW1lbnRzW2kuX3Jvd0VsZW1lbnRzLmxlbmd0aC0xXS5hZGRFdmVudExpc3RlbmVyKCJmb2N1cyIsaS5fYm90dG9tQm91bmRhcnlGb2N1c0xpc3RlbmVyKSxpLl9yZWZyZXNoUm93c0RpbWVuc2lvbnMoKSxpLl9hY2Nlc3NpYmlsaXR5VHJlZVJvb3QuYXBwZW5kQ2hpbGQoaS5fcm93Q29udGFpbmVyKSxpLl9yZW5kZXJSb3dzRGVib3VuY2VyPW5ldyBhLlRpbWVCYXNlZERlYm91bmNlcihpLl9yZW5kZXJSb3dzLmJpbmQoaSkpLGkuX3JlZnJlc2hSb3dzKCksaS5fbGl2ZVJlZ2lvbj1kb2N1bWVudC5jcmVhdGVFbGVtZW50KCJkaXYiKSxpLl9saXZlUmVnaW9uLmNsYXNzTGlzdC5hZGQoImxpdmUtcmVnaW9uIiksaS5fbGl2ZVJlZ2lvbi5zZXRBdHRyaWJ1dGUoImFyaWEtbGl2ZSIsImFzc2VydGl2ZSIpLGkuX2FjY2Vzc2liaWxpdHlUcmVlUm9vdC5hcHBlbmRDaGlsZChpLl9saXZlUmVnaW9uKSwhaS5fdGVybWluYWwuZWxlbWVudCl0aHJvdyBuZXcgRXJyb3IoIkNhbm5vdCBlbmFibGUgYWNjZXNzaWJpbGl0eSBiZWZvcmUgVGVybWluYWwub3BlbiIpO3JldHVybiBpLl90ZXJtaW5hbC5lbGVtZW50Lmluc2VydEFkamFjZW50RWxlbWVudCgiYWZ0ZXJiZWdpbiIsaS5fYWNjZXNzaWJpbGl0eVRyZWVSb290KSxpLnJlZ2lzdGVyKGkuX3JlbmRlclJvd3NEZWJvdW5jZXIpLGkucmVnaXN0ZXIoaS5fdGVybWluYWwub25SZXNpemUoKGZ1bmN0aW9uKGUpe3JldHVybiBpLl9vblJlc2l6ZShlLnJvd3MpfSkpKSxpLnJlZ2lzdGVyKGkuX3Rlcm1pbmFsLm9uUmVuZGVyKChmdW5jdGlvbihlKXtyZXR1cm4gaS5fcmVmcmVzaFJvd3MoZS5zdGFydCxlLmVuZCl9KSkpLGkucmVnaXN0ZXIoaS5fdGVybWluYWwub25TY3JvbGwoKGZ1bmN0aW9uKCl7cmV0dXJuIGkuX3JlZnJlc2hSb3dzKCl9KSkpLGkucmVnaXN0ZXIoaS5fdGVybWluYWwub25BMTF5Q2hhcigoZnVuY3Rpb24oZSl7cmV0dXJuIGkuX29uQ2hhcihlKX0pKSksaS5yZWdpc3RlcihpLl90ZXJtaW5hbC5vbkxpbmVGZWVkKChmdW5jdGlvbigpe3JldHVybiBpLl9vbkNoYXIoIlxuIil9KSkpLGkucmVnaXN0ZXIoaS5fdGVybWluYWwub25BMTF5VGFiKChmdW5jdGlvbihlKXtyZXR1cm4gaS5fb25UYWIoZSl9KSkpLGkucmVnaXN0ZXIoaS5fdGVybWluYWwub25LZXkoKGZ1bmN0aW9uKGUpe3JldHVybiBpLl9vbktleShlLmtleSl9KSkpLGkucmVnaXN0ZXIoaS5fdGVybWluYWwub25CbHVyKChmdW5jdGlvbigpe3JldHVybiBpLl9jbGVhckxpdmVSZWdpb24oKX0pKSksaS5yZWdpc3RlcihpLl9yZW5kZXJTZXJ2aWNlLm9uRGltZW5zaW9uc0NoYW5nZSgoZnVuY3Rpb24oKXtyZXR1cm4gaS5fcmVmcmVzaFJvd3NEaW1lbnNpb25zKCl9KSkpLGkuX3NjcmVlbkRwck1vbml0b3I9bmV3IHUuU2NyZWVuRHByTW9uaXRvcixpLnJlZ2lzdGVyKGkuX3NjcmVlbkRwck1vbml0b3IpLGkuX3NjcmVlbkRwck1vbml0b3Iuc2V0TGlzdGVuZXIoKGZ1bmN0aW9uKCl7cmV0dXJuIGkuX3JlZnJlc2hSb3dzRGltZW5zaW9ucygpfSkpLGkucmVnaXN0ZXIoKDAsYy5hZGREaXNwb3NhYmxlRG9tTGlzdGVuZXIpKHdpbmRvdywicmVzaXplIiwoZnVuY3Rpb24oKXtyZXR1cm4gaS5fcmVmcmVzaFJvd3NEaW1lbnNpb25zKCl9KSkpLGl9cmV0dXJuIG4odCxlKSx0LnByb3RvdHlwZS5kaXNwb3NlPWZ1bmN0aW9uKCl7ZS5wcm90b3R5cGUuZGlzcG9zZS5jYWxsKHRoaXMpLCgwLGgucmVtb3ZlRWxlbWVudEZyb21QYXJlbnQpKHRoaXMuX2FjY2Vzc2liaWxpdHlUcmVlUm9vdCksdGhpcy5fcm93RWxlbWVudHMubGVuZ3RoPTB9LHQucHJvdG90eXBlLl9vbkJvdW5kYXJ5Rm9jdXM9ZnVuY3Rpb24oZSx0KXt2YXIgcj1lLnRhcmdldCxpPXRoaXMuX3Jvd0VsZW1lbnRzWzA9PT10PzE6dGhpcy5fcm93RWxlbWVudHMubGVuZ3RoLTJdO2lmKHIuZ2V0QXR0cmlidXRlKCJhcmlhLXBvc2luc2V0IikhPT0oMD09PXQ/IjEiOiIiK3RoaXMuX3Rlcm1pbmFsLmJ1ZmZlci5saW5lcy5sZW5ndGgpJiZlLnJlbGF0ZWRUYXJnZXQ9PT1pKXt2YXIgbixvO2lmKDA9PT10PyhuPXIsbz10aGlzLl9yb3dFbGVtZW50cy5wb3AoKSx0aGlzLl9yb3dDb250YWluZXIucmVtb3ZlQ2hpbGQobykpOihuPXRoaXMuX3Jvd0VsZW1lbnRzLnNoaWZ0KCksbz1yLHRoaXMuX3Jvd0NvbnRhaW5lci5yZW1vdmVDaGlsZChuKSksbi5yZW1vdmVFdmVudExpc3RlbmVyKCJmb2N1cyIsdGhpcy5fdG9wQm91bmRhcnlGb2N1c0xpc3RlbmVyKSxvLnJlbW92ZUV2ZW50TGlzdGVuZXIoImZvY3VzIix0aGlzLl9ib3R0b21Cb3VuZGFyeUZvY3VzTGlzdGVuZXIpLDA9PT10KXt2YXIgcz10aGlzLl9jcmVhdGVBY2Nlc3NpYmlsaXR5VHJlZU5vZGUoKTt0aGlzLl9yb3dFbGVtZW50cy51bnNoaWZ0KHMpLHRoaXMuX3Jvd0NvbnRhaW5lci5pbnNlcnRBZGphY2VudEVsZW1lbnQoImFmdGVyYmVnaW4iLHMpfWVsc2Ugcz10aGlzLl9jcmVhdGVBY2Nlc3NpYmlsaXR5VHJlZU5vZGUoKSx0aGlzLl9yb3dFbGVtZW50cy5wdXNoKHMpLHRoaXMuX3Jvd0NvbnRhaW5lci5hcHBlbmRDaGlsZChzKTt0aGlzLl9yb3dFbGVtZW50c1swXS5hZGRFdmVudExpc3RlbmVyKCJmb2N1cyIsdGhpcy5fdG9wQm91bmRhcnlGb2N1c0xpc3RlbmVyKSx0aGlzLl9yb3dFbGVtZW50c1t0aGlzLl9yb3dFbGVtZW50cy5sZW5ndGgtMV0uYWRkRXZlbnRMaXN0ZW5lcigiZm9jdXMiLHRoaXMuX2JvdHRvbUJvdW5kYXJ5Rm9jdXNMaXN0ZW5lciksdGhpcy5fdGVybWluYWwuc2Nyb2xsTGluZXMoMD09PXQ/LTE6MSksdGhpcy5fcm93RWxlbWVudHNbMD09PXQ/MTp0aGlzLl9yb3dFbGVtZW50cy5sZW5ndGgtMl0uZm9jdXMoKSxlLnByZXZlbnREZWZhdWx0KCksZS5zdG9wSW1tZWRpYXRlUHJvcGFnYXRpb24oKX19LHQucHJvdG90eXBlLl9vblJlc2l6ZT1mdW5jdGlvbihlKXt0aGlzLl9yb3dFbGVtZW50c1t0aGlzLl9yb3dFbGVtZW50cy5sZW5ndGgtMV0ucmVtb3ZlRXZlbnRMaXN0ZW5lcigiZm9jdXMiLHRoaXMuX2JvdHRvbUJvdW5kYXJ5Rm9jdXNMaXN0ZW5lcik7Zm9yKHZhciB0PXRoaXMuX3Jvd0NvbnRhaW5lci5jaGlsZHJlbi5sZW5ndGg7dDx0aGlzLl90ZXJtaW5hbC5yb3dzO3QrKyl0aGlzLl9yb3dFbGVtZW50c1t0XT10aGlzLl9jcmVhdGVBY2Nlc3NpYmlsaXR5VHJlZU5vZGUoKSx0aGlzLl9yb3dDb250YWluZXIuYXBwZW5kQ2hpbGQodGhpcy5fcm93RWxlbWVudHNbdF0pO2Zvcig7dGhpcy5fcm93RWxlbWVudHMubGVuZ3RoPmU7KXRoaXMuX3Jvd0NvbnRhaW5lci5yZW1vdmVDaGlsZCh0aGlzLl9yb3dFbGVtZW50cy5wb3AoKSk7dGhpcy5fcm93RWxlbWVudHNbdGhpcy5fcm93RWxlbWVudHMubGVuZ3RoLTFdLmFkZEV2ZW50TGlzdGVuZXIoImZvY3VzIix0aGlzLl9ib3R0b21Cb3VuZGFyeUZvY3VzTGlzdGVuZXIpLHRoaXMuX3JlZnJlc2hSb3dzRGltZW5zaW9ucygpfSx0LnByb3RvdHlwZS5fY3JlYXRlQWNjZXNzaWJpbGl0eVRyZWVOb2RlPWZ1bmN0aW9uKCl7dmFyIGU9ZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgiZGl2Iik7cmV0dXJuIGUuc2V0QXR0cmlidXRlKCJyb2xlIiwibGlzdGl0ZW0iKSxlLnRhYkluZGV4PS0xLHRoaXMuX3JlZnJlc2hSb3dEaW1lbnNpb25zKGUpLGV9LHQucHJvdG90eXBlLl9vblRhYj1mdW5jdGlvbihlKXtmb3IodmFyIHQ9MDt0PGU7dCsrKXRoaXMuX29uQ2hhcigiICIpfSx0LnByb3RvdHlwZS5fb25DaGFyPWZ1bmN0aW9uKGUpe3ZhciB0PXRoaXM7dGhpcy5fbGl2ZVJlZ2lvbkxpbmVDb3VudDwyMSYmKHRoaXMuX2NoYXJzVG9Db25zdW1lLmxlbmd0aD4wP3RoaXMuX2NoYXJzVG9Db25zdW1lLnNoaWZ0KCkhPT1lJiYodGhpcy5fY2hhcnNUb0Fubm91bmNlKz1lKTp0aGlzLl9jaGFyc1RvQW5ub3VuY2UrPWUsIlxuIj09PWUmJih0aGlzLl9saXZlUmVnaW9uTGluZUNvdW50KyssMjE9PT10aGlzLl9saXZlUmVnaW9uTGluZUNvdW50JiYodGhpcy5fbGl2ZVJlZ2lvbi50ZXh0Q29udGVudCs9by50b29NdWNoT3V0cHV0KSkscy5pc01hYyYmdGhpcy5fbGl2ZVJlZ2lvbi50ZXh0Q29udGVudCYmdGhpcy5fbGl2ZVJlZ2lvbi50ZXh0Q29udGVudC5sZW5ndGg+MCYmIXRoaXMuX2xpdmVSZWdpb24ucGFyZW50Tm9kZSYmc2V0VGltZW91dCgoZnVuY3Rpb24oKXt0Ll9hY2Nlc3NpYmlsaXR5VHJlZVJvb3QuYXBwZW5kQ2hpbGQodC5fbGl2ZVJlZ2lvbil9KSwwKSl9LHQucHJvdG90eXBlLl9jbGVhckxpdmVSZWdpb249ZnVuY3Rpb24oKXt0aGlzLl9saXZlUmVnaW9uLnRleHRDb250ZW50PSIiLHRoaXMuX2xpdmVSZWdpb25MaW5lQ291bnQ9MCxzLmlzTWFjJiYoMCxoLnJlbW92ZUVsZW1lbnRGcm9tUGFyZW50KSh0aGlzLl9saXZlUmVnaW9uKX0sdC5wcm90b3R5cGUuX29uS2V5PWZ1bmN0aW9uKGUpe3RoaXMuX2NsZWFyTGl2ZVJlZ2lvbigpLHRoaXMuX2NoYXJzVG9Db25zdW1lLnB1c2goZSl9LHQucHJvdG90eXBlLl9yZWZyZXNoUm93cz1mdW5jdGlvbihlLHQpe3RoaXMuX3JlbmRlclJvd3NEZWJvdW5jZXIucmVmcmVzaChlLHQsdGhpcy5fdGVybWluYWwucm93cyl9LHQucHJvdG90eXBlLl9yZW5kZXJSb3dzPWZ1bmN0aW9uKGUsdCl7Zm9yKHZhciByPXRoaXMuX3Rlcm1pbmFsLmJ1ZmZlcixpPXIubGluZXMubGVuZ3RoLnRvU3RyaW5nKCksbj1lO248PXQ7bisrKXt2YXIgbz1yLnRyYW5zbGF0ZUJ1ZmZlckxpbmVUb1N0cmluZyhyLnlkaXNwK24sITApLHM9KHIueWRpc3ArbisxKS50b1N0cmluZygpLGE9dGhpcy5fcm93RWxlbWVudHNbbl07YSYmKDA9PT1vLmxlbmd0aD9hLmlubmVyVGV4dD0iwqAiOmEudGV4dENvbnRlbnQ9byxhLnNldEF0dHJpYnV0ZSgiYXJpYS1wb3NpbnNldCIscyksYS5zZXRBdHRyaWJ1dGUoImFyaWEtc2V0c2l6ZSIsaSkpfXRoaXMuX2Fubm91bmNlQ2hhcmFjdGVycygpfSx0LnByb3RvdHlwZS5fcmVmcmVzaFJvd3NEaW1lbnNpb25zPWZ1bmN0aW9uKCl7aWYodGhpcy5fcmVuZGVyU2VydmljZS5kaW1lbnNpb25zLmFjdHVhbENlbGxIZWlnaHQpe3RoaXMuX3Jvd0VsZW1lbnRzLmxlbmd0aCE9PXRoaXMuX3Rlcm1pbmFsLnJvd3MmJnRoaXMuX29uUmVzaXplKHRoaXMuX3Rlcm1pbmFsLnJvd3MpO2Zvcih2YXIgZT0wO2U8dGhpcy5fdGVybWluYWwucm93cztlKyspdGhpcy5fcmVmcmVzaFJvd0RpbWVuc2lvbnModGhpcy5fcm93RWxlbWVudHNbZV0pfX0sdC5wcm90b3R5cGUuX3JlZnJlc2hSb3dEaW1lbnNpb25zPWZ1bmN0aW9uKGUpe2Uuc3R5bGUuaGVpZ2h0PXRoaXMuX3JlbmRlclNlcnZpY2UuZGltZW5zaW9ucy5hY3R1YWxDZWxsSGVpZ2h0KyJweCJ9LHQucHJvdG90eXBlLl9hbm5vdW5jZUNoYXJhY3RlcnM9ZnVuY3Rpb24oKXswIT09dGhpcy5fY2hhcnNUb0Fubm91bmNlLmxlbmd0aCYmKHRoaXMuX2xpdmVSZWdpb24udGV4dENvbnRlbnQrPXRoaXMuX2NoYXJzVG9Bbm5vdW5jZSx0aGlzLl9jaGFyc1RvQW5ub3VuY2U9IiIpfSx0fShsLkRpc3Bvc2FibGUpO3QuQWNjZXNzaWJpbGl0eU1hbmFnZXI9Zn0sMzYxNDooZSx0KT0+e2Z1bmN0aW9uIHIoZSl7cmV0dXJuIGUucmVwbGFjZSgvXHI/XG4vZywiXHIiKX1mdW5jdGlvbiBpKGUsdCl7cmV0dXJuIHQ/IhtbMjAwfiIrZSsiG1syMDF+IjplfWZ1bmN0aW9uIG4oZSx0LG4pe2U9aShlPXIoZSksbi5kZWNQcml2YXRlTW9kZXMuYnJhY2tldGVkUGFzdGVNb2RlKSxuLnRyaWdnZXJEYXRhRXZlbnQoZSwhMCksdC52YWx1ZT0iIn1mdW5jdGlvbiBvKGUsdCxyKXt2YXIgaT1yLmdldEJvdW5kaW5nQ2xpZW50UmVjdCgpLG49ZS5jbGllbnRYLWkubGVmdC0xMCxvPWUuY2xpZW50WS1pLnRvcC0xMDt0LnN0eWxlLndpZHRoPSIyMHB4Iix0LnN0eWxlLmhlaWdodD0iMjBweCIsdC5zdHlsZS5sZWZ0PW4rInB4Iix0LnN0eWxlLnRvcD1vKyJweCIsdC5zdHlsZS56SW5kZXg9IjEwMDAiLHQuZm9jdXMoKX1PYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC5yaWdodENsaWNrSGFuZGxlcj10Lm1vdmVUZXh0QXJlYVVuZGVyTW91c2VDdXJzb3I9dC5wYXN0ZT10LmhhbmRsZVBhc3RlRXZlbnQ9dC5jb3B5SGFuZGxlcj10LmJyYWNrZXRUZXh0Rm9yUGFzdGU9dC5wcmVwYXJlVGV4dEZvclRlcm1pbmFsPXZvaWQgMCx0LnByZXBhcmVUZXh0Rm9yVGVybWluYWw9cix0LmJyYWNrZXRUZXh0Rm9yUGFzdGU9aSx0LmNvcHlIYW5kbGVyPWZ1bmN0aW9uKGUsdCl7ZS5jbGlwYm9hcmREYXRhJiZlLmNsaXBib2FyZERhdGEuc2V0RGF0YSgidGV4dC9wbGFpbiIsdC5zZWxlY3Rpb25UZXh0KSxlLnByZXZlbnREZWZhdWx0KCl9LHQuaGFuZGxlUGFzdGVFdmVudD1mdW5jdGlvbihlLHQscil7ZS5zdG9wUHJvcGFnYXRpb24oKSxlLmNsaXBib2FyZERhdGEmJm4oZS5jbGlwYm9hcmREYXRhLmdldERhdGEoInRleHQvcGxhaW4iKSx0LHIpfSx0LnBhc3RlPW4sdC5tb3ZlVGV4dEFyZWFVbmRlck1vdXNlQ3Vyc29yPW8sdC5yaWdodENsaWNrSGFuZGxlcj1mdW5jdGlvbihlLHQscixpLG4pe28oZSx0LHIpLG4mJmkucmlnaHRDbGlja1NlbGVjdChlKSx0LnZhbHVlPWkuc2VsZWN0aW9uVGV4dCx0LnNlbGVjdCgpfX0sNDc3NDooZSx0KT0+e3ZhciByLGksbixvO2Z1bmN0aW9uIHMoZSl7dmFyIHQ9ZS50b1N0cmluZygxNik7cmV0dXJuIHQubGVuZ3RoPDI/IjAiK3Q6dH1mdW5jdGlvbiBhKGUsdCl7cmV0dXJuIGU8dD8odCsuMDUpLyhlKy4wNSk6KGUrLjA1KS8odCsuMDUpfU9iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LCJfX2VzTW9kdWxlIix7dmFsdWU6ITB9KSx0LmNvbnRyYXN0UmF0aW89dC50b1BhZGRlZEhleD10LnJnYmE9dC5yZ2I9dC5jc3M9dC5jb2xvcj10LmNoYW5uZWxzPXZvaWQgMCxmdW5jdGlvbihlKXtlLnRvQ3NzPWZ1bmN0aW9uKGUsdCxyLGkpe3JldHVybiB2b2lkIDAhPT1pPyIjIitzKGUpK3ModCkrcyhyKStzKGkpOiIjIitzKGUpK3ModCkrcyhyKX0sZS50b1JnYmE9ZnVuY3Rpb24oZSx0LHIsaSl7cmV0dXJuIHZvaWQgMD09PWkmJihpPTI1NSksKGU8PDI0fHQ8PDE2fHI8PDh8aSk+Pj4wfX0ocj10LmNoYW5uZWxzfHwodC5jaGFubmVscz17fSkpLChpPXQuY29sb3J8fCh0LmNvbG9yPXt9KSkuYmxlbmQ9ZnVuY3Rpb24oZSx0KXt2YXIgaT0oMjU1JnQucmdiYSkvMjU1O2lmKDE9PT1pKXJldHVybntjc3M6dC5jc3MscmdiYTp0LnJnYmF9O3ZhciBuPXQucmdiYT4+MjQmMjU1LG89dC5yZ2JhPj4xNiYyNTUscz10LnJnYmE+PjgmMjU1LGE9ZS5yZ2JhPj4yNCYyNTUsYz1lLnJnYmE+PjE2JjI1NSxsPWUucmdiYT4+OCYyNTUsdT1hK01hdGgucm91bmQoKG4tYSkqaSksaD1jK01hdGgucm91bmQoKG8tYykqaSksZj1sK01hdGgucm91bmQoKHMtbCkqaSk7cmV0dXJue2NzczpyLnRvQ3NzKHUsaCxmKSxyZ2JhOnIudG9SZ2JhKHUsaCxmKX19LGkuaXNPcGFxdWU9ZnVuY3Rpb24oZSl7cmV0dXJuIDI1NT09KDI1NSZlLnJnYmEpfSxpLmVuc3VyZUNvbnRyYXN0UmF0aW89ZnVuY3Rpb24oZSx0LHIpe3ZhciBpPW8uZW5zdXJlQ29udHJhc3RSYXRpbyhlLnJnYmEsdC5yZ2JhLHIpO2lmKGkpcmV0dXJuIG8udG9Db2xvcihpPj4yNCYyNTUsaT4+MTYmMjU1LGk+PjgmMjU1KX0saS5vcGFxdWU9ZnVuY3Rpb24oZSl7dmFyIHQ9KDI1NXxlLnJnYmEpPj4+MCxpPW8udG9DaGFubmVscyh0KSxuPWlbMF0scz1pWzFdLGE9aVsyXTtyZXR1cm57Y3NzOnIudG9Dc3MobixzLGEpLHJnYmE6dH19LGkub3BhY2l0eT1mdW5jdGlvbihlLHQpe3ZhciBpPU1hdGgucm91bmQoMjU1KnQpLG49by50b0NoYW5uZWxzKGUucmdiYSkscz1uWzBdLGE9blsxXSxjPW5bMl07cmV0dXJue2NzczpyLnRvQ3NzKHMsYSxjLGkpLHJnYmE6ci50b1JnYmEocyxhLGMsaSl9fSxpLnRvQ29sb3JSR0I9ZnVuY3Rpb24oZSl7cmV0dXJuW2UucmdiYT4+MjQmMjU1LGUucmdiYT4+MTYmMjU1LGUucmdiYT4+OCYyNTVdfSwodC5jc3N8fCh0LmNzcz17fSkpLnRvQ29sb3I9ZnVuY3Rpb24oZSl7c3dpdGNoKGUubGVuZ3RoKXtjYXNlIDc6cmV0dXJue2NzczplLHJnYmE6KHBhcnNlSW50KGUuc2xpY2UoMSksMTYpPDw4fDI1NSk+Pj4wfTtjYXNlIDk6cmV0dXJue2NzczplLHJnYmE6cGFyc2VJbnQoZS5zbGljZSgxKSwxNik+Pj4wfX10aHJvdyBuZXcgRXJyb3IoImNzcy50b0NvbG9yOiBVbnN1cHBvcnRlZCBjc3MgZm9ybWF0Iil9LGZ1bmN0aW9uKGUpe2Z1bmN0aW9uIHQoZSx0LHIpe3ZhciBpPWUvMjU1LG49dC8yNTUsbz1yLzI1NTtyZXR1cm4uMjEyNiooaTw9LjAzOTI4P2kvMTIuOTI6TWF0aC5wb3coKGkrLjA1NSkvMS4wNTUsMi40KSkrLjcxNTIqKG48PS4wMzkyOD9uLzEyLjkyOk1hdGgucG93KChuKy4wNTUpLzEuMDU1LDIuNCkpKy4wNzIyKihvPD0uMDM5Mjg/by8xMi45MjpNYXRoLnBvdygobysuMDU1KS8xLjA1NSwyLjQpKX1lLnJlbGF0aXZlTHVtaW5hbmNlPWZ1bmN0aW9uKGUpe3JldHVybiB0KGU+PjE2JjI1NSxlPj44JjI1NSwyNTUmZSl9LGUucmVsYXRpdmVMdW1pbmFuY2UyPXR9KG49dC5yZ2J8fCh0LnJnYj17fSkpLGZ1bmN0aW9uKGUpe2Z1bmN0aW9uIHQoZSx0LHIpe2Zvcih2YXIgaT1lPj4yNCYyNTUsbz1lPj4xNiYyNTUscz1lPj44JjI1NSxjPXQ+PjI0JjI1NSxsPXQ+PjE2JjI1NSx1PXQ+PjgmMjU1LGg9YShuLnJlbGF0aXZlTHVtaW5hbmNlMihjLHUsbCksbi5yZWxhdGl2ZUx1bWluYW5jZTIoaSxvLHMpKTtoPHImJihjPjB8fGw+MHx8dT4wKTspYy09TWF0aC5tYXgoMCxNYXRoLmNlaWwoLjEqYykpLGwtPU1hdGgubWF4KDAsTWF0aC5jZWlsKC4xKmwpKSx1LT1NYXRoLm1heCgwLE1hdGguY2VpbCguMSp1KSksaD1hKG4ucmVsYXRpdmVMdW1pbmFuY2UyKGMsdSxsKSxuLnJlbGF0aXZlTHVtaW5hbmNlMihpLG8scykpO3JldHVybihjPDwyNHxsPDwxNnx1PDw4fDI1NSk+Pj4wfWZ1bmN0aW9uIGkoZSx0LHIpe2Zvcih2YXIgaT1lPj4yNCYyNTUsbz1lPj4xNiYyNTUscz1lPj44JjI1NSxjPXQ+PjI0JjI1NSxsPXQ+PjE2JjI1NSx1PXQ+PjgmMjU1LGg9YShuLnJlbGF0aXZlTHVtaW5hbmNlMihjLHUsbCksbi5yZWxhdGl2ZUx1bWluYW5jZTIoaSxvLHMpKTtoPHImJihjPDI1NXx8bDwyNTV8fHU8MjU1KTspYz1NYXRoLm1pbigyNTUsYytNYXRoLmNlaWwoLjEqKDI1NS1jKSkpLGw9TWF0aC5taW4oMjU1LGwrTWF0aC5jZWlsKC4xKigyNTUtbCkpKSx1PU1hdGgubWluKDI1NSx1K01hdGguY2VpbCguMSooMjU1LXUpKSksaD1hKG4ucmVsYXRpdmVMdW1pbmFuY2UyKGMsdSxsKSxuLnJlbGF0aXZlTHVtaW5hbmNlMihpLG8scykpO3JldHVybihjPDwyNHxsPDwxNnx1PDw4fDI1NSk+Pj4wfWUuZW5zdXJlQ29udHJhc3RSYXRpbz1mdW5jdGlvbihlLHIsbyl7dmFyIHM9bi5yZWxhdGl2ZUx1bWluYW5jZShlPj44KSxjPW4ucmVsYXRpdmVMdW1pbmFuY2Uocj4+OCk7aWYoYShzLGMpPG8pcmV0dXJuIGM8cz90KGUscixvKTppKGUscixvKX0sZS5yZWR1Y2VMdW1pbmFuY2U9dCxlLmluY3JlYXNlTHVtaW5hbmNlPWksZS50b0NoYW5uZWxzPWZ1bmN0aW9uKGUpe3JldHVybltlPj4yNCYyNTUsZT4+MTYmMjU1LGU+PjgmMjU1LDI1NSZlXX0sZS50b0NvbG9yPWZ1bmN0aW9uKGUsdCxpKXtyZXR1cm57Y3NzOnIudG9Dc3MoZSx0LGkpLHJnYmE6ci50b1JnYmEoZSx0LGkpfX19KG89dC5yZ2JhfHwodC5yZ2JhPXt9KSksdC50b1BhZGRlZEhleD1zLHQuY29udHJhc3RSYXRpbz1hfSw3MjM5OihlLHQpPT57T2JqZWN0LmRlZmluZVByb3BlcnR5KHQsIl9fZXNNb2R1bGUiLHt2YWx1ZTohMH0pLHQuQ29sb3JDb250cmFzdENhY2hlPXZvaWQgMDt2YXIgcj1mdW5jdGlvbigpe2Z1bmN0aW9uIGUoKXt0aGlzLl9jb2xvcj17fSx0aGlzLl9yZ2JhPXt9fXJldHVybiBlLnByb3RvdHlwZS5jbGVhcj1mdW5jdGlvbigpe3RoaXMuX2NvbG9yPXt9LHRoaXMuX3JnYmE9e319LGUucHJvdG90eXBlLnNldENzcz1mdW5jdGlvbihlLHQscil7dGhpcy5fcmdiYVtlXXx8KHRoaXMuX3JnYmFbZV09e30pLHRoaXMuX3JnYmFbZV1bdF09cn0sZS5wcm90b3R5cGUuZ2V0Q3NzPWZ1bmN0aW9uKGUsdCl7cmV0dXJuIHRoaXMuX3JnYmFbZV0/dGhpcy5fcmdiYVtlXVt0XTp2b2lkIDB9LGUucHJvdG90eXBlLnNldENvbG9yPWZ1bmN0aW9uKGUsdCxyKXt0aGlzLl9jb2xvcltlXXx8KHRoaXMuX2NvbG9yW2VdPXt9KSx0aGlzLl9jb2xvcltlXVt0XT1yfSxlLnByb3RvdHlwZS5nZXRDb2xvcj1mdW5jdGlvbihlLHQpe3JldHVybiB0aGlzLl9jb2xvcltlXT90aGlzLl9jb2xvcltlXVt0XTp2b2lkIDB9LGV9KCk7dC5Db2xvckNvbnRyYXN0Q2FjaGU9cn0sNTY4MDpmdW5jdGlvbihlLHQscil7dmFyIGk9dGhpcyYmdGhpcy5fX3NwcmVhZEFycmF5fHxmdW5jdGlvbihlLHQscil7aWYocnx8Mj09PWFyZ3VtZW50cy5sZW5ndGgpZm9yKHZhciBpLG49MCxvPXQubGVuZ3RoO248bztuKyspIWkmJm4gaW4gdHx8KGl8fChpPUFycmF5LnByb3RvdHlwZS5zbGljZS5jYWxsKHQsMCxuKSksaVtuXT10W25dKTtyZXR1cm4gZS5jb25jYXQoaXx8QXJyYXkucHJvdG90eXBlLnNsaWNlLmNhbGwodCkpfTtPYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC5Db2xvck1hbmFnZXI9dC5ERUZBVUxUX0FOU0lfQ09MT1JTPXZvaWQgMDt2YXIgbj1yKDQ3NzQpLG89cig3MjM5KSxzPW4uY3NzLnRvQ29sb3IoIiNmZmZmZmYiKSxhPW4uY3NzLnRvQ29sb3IoIiMwMDAwMDAiKSxjPW4uY3NzLnRvQ29sb3IoIiNmZmZmZmYiKSxsPW4uY3NzLnRvQ29sb3IoIiMwMDAwMDAiKSx1PXtjc3M6InJnYmEoMjU1LCAyNTUsIDI1NSwgMC4zKSIscmdiYTo0Mjk0OTY3MTE3fTt0LkRFRkFVTFRfQU5TSV9DT0xPUlM9T2JqZWN0LmZyZWV6ZShmdW5jdGlvbigpe2Zvcih2YXIgZT1bbi5jc3MudG9Db2xvcigiIzJlMzQzNiIpLG4uY3NzLnRvQ29sb3IoIiNjYzAwMDAiKSxuLmNzcy50b0NvbG9yKCIjNGU5YTA2Iiksbi5jc3MudG9Db2xvcigiI2M0YTAwMCIpLG4uY3NzLnRvQ29sb3IoIiMzNDY1YTQiKSxuLmNzcy50b0NvbG9yKCIjNzU1MDdiIiksbi5jc3MudG9Db2xvcigiIzA2OTg5YSIpLG4uY3NzLnRvQ29sb3IoIiNkM2Q3Y2YiKSxuLmNzcy50b0NvbG9yKCIjNTU1NzUzIiksbi5jc3MudG9Db2xvcigiI2VmMjkyOSIpLG4uY3NzLnRvQ29sb3IoIiM4YWUyMzQiKSxuLmNzcy50b0NvbG9yKCIjZmNlOTRmIiksbi5jc3MudG9Db2xvcigiIzcyOWZjZiIpLG4uY3NzLnRvQ29sb3IoIiNhZDdmYTgiKSxuLmNzcy50b0NvbG9yKCIjMzRlMmUyIiksbi5jc3MudG9Db2xvcigiI2VlZWVlYyIpXSx0PVswLDk1LDEzNSwxNzUsMjE1LDI1NV0scj0wO3I8MjE2O3IrKyl7dmFyIGk9dFtyLzM2JTZ8MF0sbz10W3IvNiU2fDBdLHM9dFtyJTZdO2UucHVzaCh7Y3NzOm4uY2hhbm5lbHMudG9Dc3MoaSxvLHMpLHJnYmE6bi5jaGFubmVscy50b1JnYmEoaSxvLHMpfSl9Zm9yKHI9MDtyPDI0O3IrKyl7dmFyIGE9OCsxMCpyO2UucHVzaCh7Y3NzOm4uY2hhbm5lbHMudG9Dc3MoYSxhLGEpLHJnYmE6bi5jaGFubmVscy50b1JnYmEoYSxhLGEpfSl9cmV0dXJuIGV9KCkpO3ZhciBoPWZ1bmN0aW9uKCl7ZnVuY3Rpb24gZShlLHIpe3RoaXMuYWxsb3dUcmFuc3BhcmVuY3k9cjt2YXIgaT1lLmNyZWF0ZUVsZW1lbnQoImNhbnZhcyIpO2kud2lkdGg9MSxpLmhlaWdodD0xO3ZhciBoPWkuZ2V0Q29udGV4dCgiMmQiKTtpZighaCl0aHJvdyBuZXcgRXJyb3IoIkNvdWxkIG5vdCBnZXQgcmVuZGVyaW5nIGNvbnRleHQiKTt0aGlzLl9jdHg9aCx0aGlzLl9jdHguZ2xvYmFsQ29tcG9zaXRlT3BlcmF0aW9uPSJjb3B5Iix0aGlzLl9saXRtdXNDb2xvcj10aGlzLl9jdHguY3JlYXRlTGluZWFyR3JhZGllbnQoMCwwLDEsMSksdGhpcy5fY29udHJhc3RDYWNoZT1uZXcgby5Db2xvckNvbnRyYXN0Q2FjaGUsdGhpcy5jb2xvcnM9e2ZvcmVncm91bmQ6cyxiYWNrZ3JvdW5kOmEsY3Vyc29yOmMsY3Vyc29yQWNjZW50Omwsc2VsZWN0aW9uVHJhbnNwYXJlbnQ6dSxzZWxlY3Rpb25PcGFxdWU6bi5jb2xvci5ibGVuZChhLHUpLGFuc2k6dC5ERUZBVUxUX0FOU0lfQ09MT1JTLnNsaWNlKCksY29udHJhc3RDYWNoZTp0aGlzLl9jb250cmFzdENhY2hlfSx0aGlzLl91cGRhdGVSZXN0b3JlQ29sb3JzKCl9cmV0dXJuIGUucHJvdG90eXBlLm9uT3B0aW9uc0NoYW5nZT1mdW5jdGlvbihlKXsibWluaW11bUNvbnRyYXN0UmF0aW8iPT09ZSYmdGhpcy5fY29udHJhc3RDYWNoZS5jbGVhcigpfSxlLnByb3RvdHlwZS5zZXRUaGVtZT1mdW5jdGlvbihlKXt2b2lkIDA9PT1lJiYoZT17fSksdGhpcy5jb2xvcnMuZm9yZWdyb3VuZD10aGlzLl9wYXJzZUNvbG9yKGUuZm9yZWdyb3VuZCxzKSx0aGlzLmNvbG9ycy5iYWNrZ3JvdW5kPXRoaXMuX3BhcnNlQ29sb3IoZS5iYWNrZ3JvdW5kLGEpLHRoaXMuY29sb3JzLmN1cnNvcj10aGlzLl9wYXJzZUNvbG9yKGUuY3Vyc29yLGMsITApLHRoaXMuY29sb3JzLmN1cnNvckFjY2VudD10aGlzLl9wYXJzZUNvbG9yKGUuY3Vyc29yQWNjZW50LGwsITApLHRoaXMuY29sb3JzLnNlbGVjdGlvblRyYW5zcGFyZW50PXRoaXMuX3BhcnNlQ29sb3IoZS5zZWxlY3Rpb24sdSwhMCksdGhpcy5jb2xvcnMuc2VsZWN0aW9uT3BhcXVlPW4uY29sb3IuYmxlbmQodGhpcy5jb2xvcnMuYmFja2dyb3VuZCx0aGlzLmNvbG9ycy5zZWxlY3Rpb25UcmFuc3BhcmVudCksbi5jb2xvci5pc09wYXF1ZSh0aGlzLmNvbG9ycy5zZWxlY3Rpb25UcmFuc3BhcmVudCkmJih0aGlzLmNvbG9ycy5zZWxlY3Rpb25UcmFuc3BhcmVudD1uLmNvbG9yLm9wYWNpdHkodGhpcy5jb2xvcnMuc2VsZWN0aW9uVHJhbnNwYXJlbnQsLjMpKSx0aGlzLmNvbG9ycy5hbnNpWzBdPXRoaXMuX3BhcnNlQ29sb3IoZS5ibGFjayx0LkRFRkFVTFRfQU5TSV9DT0xPUlNbMF0pLHRoaXMuY29sb3JzLmFuc2lbMV09dGhpcy5fcGFyc2VDb2xvcihlLnJlZCx0LkRFRkFVTFRfQU5TSV9DT0xPUlNbMV0pLHRoaXMuY29sb3JzLmFuc2lbMl09dGhpcy5fcGFyc2VDb2xvcihlLmdyZWVuLHQuREVGQVVMVF9BTlNJX0NPTE9SU1syXSksdGhpcy5jb2xvcnMuYW5zaVszXT10aGlzLl9wYXJzZUNvbG9yKGUueWVsbG93LHQuREVGQVVMVF9BTlNJX0NPTE9SU1szXSksdGhpcy5jb2xvcnMuYW5zaVs0XT10aGlzLl9wYXJzZUNvbG9yKGUuYmx1ZSx0LkRFRkFVTFRfQU5TSV9DT0xPUlNbNF0pLHRoaXMuY29sb3JzLmFuc2lbNV09dGhpcy5fcGFyc2VDb2xvcihlLm1hZ2VudGEsdC5ERUZBVUxUX0FOU0lfQ09MT1JTWzVdKSx0aGlzLmNvbG9ycy5hbnNpWzZdPXRoaXMuX3BhcnNlQ29sb3IoZS5jeWFuLHQuREVGQVVMVF9BTlNJX0NPTE9SU1s2XSksdGhpcy5jb2xvcnMuYW5zaVs3XT10aGlzLl9wYXJzZUNvbG9yKGUud2hpdGUsdC5ERUZBVUxUX0FOU0lfQ09MT1JTWzddKSx0aGlzLmNvbG9ycy5hbnNpWzhdPXRoaXMuX3BhcnNlQ29sb3IoZS5icmlnaHRCbGFjayx0LkRFRkFVTFRfQU5TSV9DT0xPUlNbOF0pLHRoaXMuY29sb3JzLmFuc2lbOV09dGhpcy5fcGFyc2VDb2xvcihlLmJyaWdodFJlZCx0LkRFRkFVTFRfQU5TSV9DT0xPUlNbOV0pLHRoaXMuY29sb3JzLmFuc2lbMTBdPXRoaXMuX3BhcnNlQ29sb3IoZS5icmlnaHRHcmVlbix0LkRFRkFVTFRfQU5TSV9DT0xPUlNbMTBdKSx0aGlzLmNvbG9ycy5hbnNpWzExXT10aGlzLl9wYXJzZUNvbG9yKGUuYnJpZ2h0WWVsbG93LHQuREVGQVVMVF9BTlNJX0NPTE9SU1sxMV0pLHRoaXMuY29sb3JzLmFuc2lbMTJdPXRoaXMuX3BhcnNlQ29sb3IoZS5icmlnaHRCbHVlLHQuREVGQVVMVF9BTlNJX0NPTE9SU1sxMl0pLHRoaXMuY29sb3JzLmFuc2lbMTNdPXRoaXMuX3BhcnNlQ29sb3IoZS5icmlnaHRNYWdlbnRhLHQuREVGQVVMVF9BTlNJX0NPTE9SU1sxM10pLHRoaXMuY29sb3JzLmFuc2lbMTRdPXRoaXMuX3BhcnNlQ29sb3IoZS5icmlnaHRDeWFuLHQuREVGQVVMVF9BTlNJX0NPTE9SU1sxNF0pLHRoaXMuY29sb3JzLmFuc2lbMTVdPXRoaXMuX3BhcnNlQ29sb3IoZS5icmlnaHRXaGl0ZSx0LkRFRkFVTFRfQU5TSV9DT0xPUlNbMTVdKSx0aGlzLl9jb250cmFzdENhY2hlLmNsZWFyKCksdGhpcy5fdXBkYXRlUmVzdG9yZUNvbG9ycygpfSxlLnByb3RvdHlwZS5yZXN0b3JlQ29sb3I9ZnVuY3Rpb24oZSl7aWYodm9pZCAwIT09ZSlzd2l0Y2goZSl7Y2FzZSAyNTY6dGhpcy5jb2xvcnMuZm9yZWdyb3VuZD10aGlzLl9yZXN0b3JlQ29sb3JzLmZvcmVncm91bmQ7YnJlYWs7Y2FzZSAyNTc6dGhpcy5jb2xvcnMuYmFja2dyb3VuZD10aGlzLl9yZXN0b3JlQ29sb3JzLmJhY2tncm91bmQ7YnJlYWs7Y2FzZSAyNTg6dGhpcy5jb2xvcnMuY3Vyc29yPXRoaXMuX3Jlc3RvcmVDb2xvcnMuY3Vyc29yO2JyZWFrO2RlZmF1bHQ6dGhpcy5jb2xvcnMuYW5zaVtlXT10aGlzLl9yZXN0b3JlQ29sb3JzLmFuc2lbZV19ZWxzZSBmb3IodmFyIHQ9MDt0PHRoaXMuX3Jlc3RvcmVDb2xvcnMuYW5zaS5sZW5ndGg7Kyt0KXRoaXMuY29sb3JzLmFuc2lbdF09dGhpcy5fcmVzdG9yZUNvbG9ycy5hbnNpW3RdfSxlLnByb3RvdHlwZS5fdXBkYXRlUmVzdG9yZUNvbG9ycz1mdW5jdGlvbigpe3RoaXMuX3Jlc3RvcmVDb2xvcnM9e2ZvcmVncm91bmQ6dGhpcy5jb2xvcnMuZm9yZWdyb3VuZCxiYWNrZ3JvdW5kOnRoaXMuY29sb3JzLmJhY2tncm91bmQsY3Vyc29yOnRoaXMuY29sb3JzLmN1cnNvcixhbnNpOmkoW10sdGhpcy5jb2xvcnMuYW5zaSwhMCl9fSxlLnByb3RvdHlwZS5fcGFyc2VDb2xvcj1mdW5jdGlvbihlLHQscil7aWYodm9pZCAwPT09ciYmKHI9dGhpcy5hbGxvd1RyYW5zcGFyZW5jeSksdm9pZCAwPT09ZSlyZXR1cm4gdDtpZih0aGlzLl9jdHguZmlsbFN0eWxlPXRoaXMuX2xpdG11c0NvbG9yLHRoaXMuX2N0eC5maWxsU3R5bGU9ZSwic3RyaW5nIiE9dHlwZW9mIHRoaXMuX2N0eC5maWxsU3R5bGUpcmV0dXJuIGNvbnNvbGUud2FybigiQ29sb3I6ICIrZSsiIGlzIGludmFsaWQgdXNpbmcgZmFsbGJhY2sgIit0LmNzcyksdDt0aGlzLl9jdHguZmlsbFJlY3QoMCwwLDEsMSk7dmFyIGk9dGhpcy5fY3R4LmdldEltYWdlRGF0YSgwLDAsMSwxKS5kYXRhO2lmKDI1NSE9PWlbM10pe2lmKCFyKXJldHVybiBjb25zb2xlLndhcm4oIkNvbG9yOiAiK2UrIiBpcyB1c2luZyB0cmFuc3BhcmVuY3ksIGJ1dCBhbGxvd1RyYW5zcGFyZW5jeSBpcyBmYWxzZS4gVXNpbmcgZmFsbGJhY2sgIit0LmNzcysiLiIpLHQ7dmFyIG89dGhpcy5fY3R4LmZpbGxTdHlsZS5zdWJzdHJpbmcoNSx0aGlzLl9jdHguZmlsbFN0eWxlLmxlbmd0aC0xKS5zcGxpdCgiLCIpLm1hcCgoZnVuY3Rpb24oZSl7cmV0dXJuIE51bWJlcihlKX0pKSxzPW9bMF0sYT1vWzFdLGM9b1syXSxsPW9bM10sdT1NYXRoLnJvdW5kKDI1NSpsKTtyZXR1cm57cmdiYTpuLmNoYW5uZWxzLnRvUmdiYShzLGEsYyx1KSxjc3M6ZX19cmV0dXJue2Nzczp0aGlzLl9jdHguZmlsbFN0eWxlLHJnYmE6bi5jaGFubmVscy50b1JnYmEoaVswXSxpWzFdLGlbMl0saVszXSl9fSxlfSgpO3QuQ29sb3JNYW5hZ2VyPWh9LDk2MzE6KGUsdCk9PntPYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC5yZW1vdmVFbGVtZW50RnJvbVBhcmVudD12b2lkIDAsdC5yZW1vdmVFbGVtZW50RnJvbVBhcmVudD1mdW5jdGlvbigpe2Zvcih2YXIgZSx0PVtdLHI9MDtyPGFyZ3VtZW50cy5sZW5ndGg7cisrKXRbcl09YXJndW1lbnRzW3JdO2Zvcih2YXIgaT0wLG49dDtpPG4ubGVuZ3RoO2krKyl7dmFyIG89bltpXTtudWxsPT09KGU9bnVsbD09bz92b2lkIDA6by5wYXJlbnRFbGVtZW50KXx8dm9pZCAwPT09ZXx8ZS5yZW1vdmVDaGlsZChvKX19fSwzNjU2OihlLHQpPT57T2JqZWN0LmRlZmluZVByb3BlcnR5KHQsIl9fZXNNb2R1bGUiLHt2YWx1ZTohMH0pLHQuYWRkRGlzcG9zYWJsZURvbUxpc3RlbmVyPXZvaWQgMCx0LmFkZERpc3Bvc2FibGVEb21MaXN0ZW5lcj1mdW5jdGlvbihlLHQscixpKXtlLmFkZEV2ZW50TGlzdGVuZXIodCxyLGkpO3ZhciBuPSExO3JldHVybntkaXNwb3NlOmZ1bmN0aW9uKCl7bnx8KG49ITAsZS5yZW1vdmVFdmVudExpc3RlbmVyKHQscixpKSl9fX19LDM1NTE6ZnVuY3Rpb24oZSx0LHIpe3ZhciBpPXRoaXMmJnRoaXMuX19kZWNvcmF0ZXx8ZnVuY3Rpb24oZSx0LHIsaSl7dmFyIG4sbz1hcmd1bWVudHMubGVuZ3RoLHM9bzwzP3Q6bnVsbD09PWk/aT1PYmplY3QuZ2V0T3duUHJvcGVydHlEZXNjcmlwdG9yKHQscik6aTtpZigib2JqZWN0Ij09dHlwZW9mIFJlZmxlY3QmJiJmdW5jdGlvbiI9PXR5cGVvZiBSZWZsZWN0LmRlY29yYXRlKXM9UmVmbGVjdC5kZWNvcmF0ZShlLHQscixpKTtlbHNlIGZvcih2YXIgYT1lLmxlbmd0aC0xO2E+PTA7YS0tKShuPWVbYV0pJiYocz0obzwzP24ocyk6bz4zP24odCxyLHMpOm4odCxyKSl8fHMpO3JldHVybiBvPjMmJnMmJk9iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LHIscyksc30sbj10aGlzJiZ0aGlzLl9fcGFyYW18fGZ1bmN0aW9uKGUsdCl7cmV0dXJuIGZ1bmN0aW9uKHIsaSl7dChyLGksZSl9fTtPYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC5Nb3VzZVpvbmU9dC5MaW5raWZpZXI9dm9pZCAwO3ZhciBvPXIoODQ2MCkscz1yKDI1ODUpLGE9ZnVuY3Rpb24oKXtmdW5jdGlvbiBlKGUsdCxyKXt0aGlzLl9idWZmZXJTZXJ2aWNlPWUsdGhpcy5fbG9nU2VydmljZT10LHRoaXMuX3VuaWNvZGVTZXJ2aWNlPXIsdGhpcy5fbGlua01hdGNoZXJzPVtdLHRoaXMuX25leHRMaW5rTWF0Y2hlcklkPTAsdGhpcy5fb25TaG93TGlua1VuZGVybGluZT1uZXcgby5FdmVudEVtaXR0ZXIsdGhpcy5fb25IaWRlTGlua1VuZGVybGluZT1uZXcgby5FdmVudEVtaXR0ZXIsdGhpcy5fb25MaW5rVG9vbHRpcD1uZXcgby5FdmVudEVtaXR0ZXIsdGhpcy5fcm93c1RvTGlua2lmeT17c3RhcnQ6dm9pZCAwLGVuZDp2b2lkIDB9fXJldHVybiBPYmplY3QuZGVmaW5lUHJvcGVydHkoZS5wcm90b3R5cGUsIm9uU2hvd0xpbmtVbmRlcmxpbmUiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fb25TaG93TGlua1VuZGVybGluZS5ldmVudH0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxPYmplY3QuZGVmaW5lUHJvcGVydHkoZS5wcm90b3R5cGUsIm9uSGlkZUxpbmtVbmRlcmxpbmUiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fb25IaWRlTGlua1VuZGVybGluZS5ldmVudH0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxPYmplY3QuZGVmaW5lUHJvcGVydHkoZS5wcm90b3R5cGUsIm9uTGlua1Rvb2x0aXAiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fb25MaW5rVG9vbHRpcC5ldmVudH0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxlLnByb3RvdHlwZS5hdHRhY2hUb0RvbT1mdW5jdGlvbihlLHQpe3RoaXMuX2VsZW1lbnQ9ZSx0aGlzLl9tb3VzZVpvbmVNYW5hZ2VyPXR9LGUucHJvdG90eXBlLmxpbmtpZnlSb3dzPWZ1bmN0aW9uKHQscil7dmFyIGk9dGhpczt0aGlzLl9tb3VzZVpvbmVNYW5hZ2VyJiYodm9pZCAwPT09dGhpcy5fcm93c1RvTGlua2lmeS5zdGFydHx8dm9pZCAwPT09dGhpcy5fcm93c1RvTGlua2lmeS5lbmQ/KHRoaXMuX3Jvd3NUb0xpbmtpZnkuc3RhcnQ9dCx0aGlzLl9yb3dzVG9MaW5raWZ5LmVuZD1yKToodGhpcy5fcm93c1RvTGlua2lmeS5zdGFydD1NYXRoLm1pbih0aGlzLl9yb3dzVG9MaW5raWZ5LnN0YXJ0LHQpLHRoaXMuX3Jvd3NUb0xpbmtpZnkuZW5kPU1hdGgubWF4KHRoaXMuX3Jvd3NUb0xpbmtpZnkuZW5kLHIpKSx0aGlzLl9tb3VzZVpvbmVNYW5hZ2VyLmNsZWFyQWxsKHQsciksdGhpcy5fcm93c1RpbWVvdXRJZCYmY2xlYXJUaW1lb3V0KHRoaXMuX3Jvd3NUaW1lb3V0SWQpLHRoaXMuX3Jvd3NUaW1lb3V0SWQ9c2V0VGltZW91dCgoZnVuY3Rpb24oKXtyZXR1cm4gaS5fbGlua2lmeVJvd3MoKX0pLGUuX3RpbWVCZWZvcmVMYXRlbmN5KSl9LGUucHJvdG90eXBlLl9saW5raWZ5Um93cz1mdW5jdGlvbigpe3RoaXMuX3Jvd3NUaW1lb3V0SWQ9dm9pZCAwO3ZhciBlPXRoaXMuX2J1ZmZlclNlcnZpY2UuYnVmZmVyO2lmKHZvaWQgMCE9PXRoaXMuX3Jvd3NUb0xpbmtpZnkuc3RhcnQmJnZvaWQgMCE9PXRoaXMuX3Jvd3NUb0xpbmtpZnkuZW5kKXt2YXIgdD1lLnlkaXNwK3RoaXMuX3Jvd3NUb0xpbmtpZnkuc3RhcnQ7aWYoISh0Pj1lLmxpbmVzLmxlbmd0aCkpe2Zvcih2YXIgcj1lLnlkaXNwK01hdGgubWluKHRoaXMuX3Jvd3NUb0xpbmtpZnkuZW5kLHRoaXMuX2J1ZmZlclNlcnZpY2Uucm93cykrMSxpPU1hdGguY2VpbCgyZTMvdGhpcy5fYnVmZmVyU2VydmljZS5jb2xzKSxuPXRoaXMuX2J1ZmZlclNlcnZpY2UuYnVmZmVyLml0ZXJhdG9yKCExLHQscixpLGkpO24uaGFzTmV4dCgpOylmb3IodmFyIG89bi5uZXh0KCkscz0wO3M8dGhpcy5fbGlua01hdGNoZXJzLmxlbmd0aDtzKyspdGhpcy5fZG9MaW5raWZ5Um93KG8ucmFuZ2UuZmlyc3Qsby5jb250ZW50LHRoaXMuX2xpbmtNYXRjaGVyc1tzXSk7dGhpcy5fcm93c1RvTGlua2lmeS5zdGFydD12b2lkIDAsdGhpcy5fcm93c1RvTGlua2lmeS5lbmQ9dm9pZCAwfX1lbHNlIHRoaXMuX2xvZ1NlcnZpY2UuZGVidWcoIl9yb3dUb0xpbmtpZnkgd2FzIHVuc2V0IGJlZm9yZSBfbGlua2lmeVJvd3Mgd2FzIGNhbGxlZCIpfSxlLnByb3RvdHlwZS5yZWdpc3RlckxpbmtNYXRjaGVyPWZ1bmN0aW9uKGUsdCxyKXtpZih2b2lkIDA9PT1yJiYocj17fSksIXQpdGhyb3cgbmV3IEVycm9yKCJoYW5kbGVyIG11c3QgYmUgZGVmaW5lZCIpO3ZhciBpPXtpZDp0aGlzLl9uZXh0TGlua01hdGNoZXJJZCsrLHJlZ2V4OmUsaGFuZGxlcjp0LG1hdGNoSW5kZXg6ci5tYXRjaEluZGV4LHZhbGlkYXRpb25DYWxsYmFjazpyLnZhbGlkYXRpb25DYWxsYmFjayxob3ZlclRvb2x0aXBDYWxsYmFjazpyLnRvb2x0aXBDYWxsYmFjayxob3ZlckxlYXZlQ2FsbGJhY2s6ci5sZWF2ZUNhbGxiYWNrLHdpbGxMaW5rQWN0aXZhdGU6ci53aWxsTGlua0FjdGl2YXRlLHByaW9yaXR5OnIucHJpb3JpdHl8fDB9O3JldHVybiB0aGlzLl9hZGRMaW5rTWF0Y2hlclRvTGlzdChpKSxpLmlkfSxlLnByb3RvdHlwZS5fYWRkTGlua01hdGNoZXJUb0xpc3Q9ZnVuY3Rpb24oZSl7aWYoMCE9PXRoaXMuX2xpbmtNYXRjaGVycy5sZW5ndGgpe2Zvcih2YXIgdD10aGlzLl9saW5rTWF0Y2hlcnMubGVuZ3RoLTE7dD49MDt0LS0paWYoZS5wcmlvcml0eTw9dGhpcy5fbGlua01hdGNoZXJzW3RdLnByaW9yaXR5KXJldHVybiB2b2lkIHRoaXMuX2xpbmtNYXRjaGVycy5zcGxpY2UodCsxLDAsZSk7dGhpcy5fbGlua01hdGNoZXJzLnNwbGljZSgwLDAsZSl9ZWxzZSB0aGlzLl9saW5rTWF0Y2hlcnMucHVzaChlKX0sZS5wcm90b3R5cGUuZGVyZWdpc3RlckxpbmtNYXRjaGVyPWZ1bmN0aW9uKGUpe2Zvcih2YXIgdD0wO3Q8dGhpcy5fbGlua01hdGNoZXJzLmxlbmd0aDt0KyspaWYodGhpcy5fbGlua01hdGNoZXJzW3RdLmlkPT09ZSlyZXR1cm4gdGhpcy5fbGlua01hdGNoZXJzLnNwbGljZSh0LDEpLCEwO3JldHVybiExfSxlLnByb3RvdHlwZS5fZG9MaW5raWZ5Um93PWZ1bmN0aW9uKGUsdCxyKXtmb3IodmFyIGksbj10aGlzLG89bmV3IFJlZ0V4cChyLnJlZ2V4LnNvdXJjZSwoci5yZWdleC5mbGFnc3x8IiIpKyJnIikscz0tMSxhPWZ1bmN0aW9uKCl7dmFyIGE9aVsibnVtYmVyIiE9dHlwZW9mIHIubWF0Y2hJbmRleD8wOnIubWF0Y2hJbmRleF07aWYoIWEpcmV0dXJuIGMuX2xvZ1NlcnZpY2UuZGVidWcoIm1hdGNoIGZvdW5kIHdpdGhvdXQgY29ycmVzcG9uZGluZyBtYXRjaEluZGV4IixpLHIpLCJicmVhayI7aWYocz10LmluZGV4T2YoYSxzKzEpLG8ubGFzdEluZGV4PXMrYS5sZW5ndGgsczwwKXJldHVybiJicmVhayI7dmFyIGw9Yy5fYnVmZmVyU2VydmljZS5idWZmZXIuc3RyaW5nSW5kZXhUb0J1ZmZlckluZGV4KGUscyk7aWYobFswXTwwKXJldHVybiJicmVhayI7dmFyIHU9Yy5fYnVmZmVyU2VydmljZS5idWZmZXIubGluZXMuZ2V0KGxbMF0pO2lmKCF1KXJldHVybiJicmVhayI7dmFyIGg9dS5nZXRGZyhsWzFdKSxmPWg/aD4+OSY1MTE6dm9pZCAwO3IudmFsaWRhdGlvbkNhbGxiYWNrP3IudmFsaWRhdGlvbkNhbGxiYWNrKGEsKGZ1bmN0aW9uKGUpe24uX3Jvd3NUaW1lb3V0SWR8fGUmJm4uX2FkZExpbmsobFsxXSxsWzBdLW4uX2J1ZmZlclNlcnZpY2UuYnVmZmVyLnlkaXNwLGEscixmKX0pKTpjLl9hZGRMaW5rKGxbMV0sbFswXS1jLl9idWZmZXJTZXJ2aWNlLmJ1ZmZlci55ZGlzcCxhLHIsZil9LGM9dGhpcztudWxsIT09KGk9by5leGVjKHQpKSYmImJyZWFrIiE9PWEoKTspO30sZS5wcm90b3R5cGUuX2FkZExpbms9ZnVuY3Rpb24oZSx0LHIsaSxuKXt2YXIgbz10aGlzO2lmKHRoaXMuX21vdXNlWm9uZU1hbmFnZXImJnRoaXMuX2VsZW1lbnQpe3ZhciBzPXRoaXMuX3VuaWNvZGVTZXJ2aWNlLmdldFN0cmluZ0NlbGxXaWR0aChyKSxhPWUldGhpcy5fYnVmZmVyU2VydmljZS5jb2xzLGw9dCtNYXRoLmZsb29yKGUvdGhpcy5fYnVmZmVyU2VydmljZS5jb2xzKSx1PShhK3MpJXRoaXMuX2J1ZmZlclNlcnZpY2UuY29scyxoPWwrTWF0aC5mbG9vcigoYStzKS90aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHMpOzA9PT11JiYodT10aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHMsaC0tKSx0aGlzLl9tb3VzZVpvbmVNYW5hZ2VyLmFkZChuZXcgYyhhKzEsbCsxLHUrMSxoKzEsKGZ1bmN0aW9uKGUpe2lmKGkuaGFuZGxlcilyZXR1cm4gaS5oYW5kbGVyKGUscik7dmFyIHQ9d2luZG93Lm9wZW4oKTt0Pyh0Lm9wZW5lcj1udWxsLHQubG9jYXRpb24uaHJlZj1yKTpjb25zb2xlLndhcm4oIk9wZW5pbmcgbGluayBibG9ja2VkIGFzIG9wZW5lciBjb3VsZCBub3QgYmUgY2xlYXJlZCIpfSksKGZ1bmN0aW9uKCl7by5fb25TaG93TGlua1VuZGVybGluZS5maXJlKG8uX2NyZWF0ZUxpbmtIb3ZlckV2ZW50KGEsbCx1LGgsbikpLG8uX2VsZW1lbnQuY2xhc3NMaXN0LmFkZCgieHRlcm0tY3Vyc29yLXBvaW50ZXIiKX0pLChmdW5jdGlvbihlKXtvLl9vbkxpbmtUb29sdGlwLmZpcmUoby5fY3JlYXRlTGlua0hvdmVyRXZlbnQoYSxsLHUsaCxuKSksaS5ob3ZlclRvb2x0aXBDYWxsYmFjayYmaS5ob3ZlclRvb2x0aXBDYWxsYmFjayhlLHIse3N0YXJ0Ont4OmEseTpsfSxlbmQ6e3g6dSx5Omh9fSl9KSwoZnVuY3Rpb24oKXtvLl9vbkhpZGVMaW5rVW5kZXJsaW5lLmZpcmUoby5fY3JlYXRlTGlua0hvdmVyRXZlbnQoYSxsLHUsaCxuKSksby5fZWxlbWVudC5jbGFzc0xpc3QucmVtb3ZlKCJ4dGVybS1jdXJzb3ItcG9pbnRlciIpLGkuaG92ZXJMZWF2ZUNhbGxiYWNrJiZpLmhvdmVyTGVhdmVDYWxsYmFjaygpfSksKGZ1bmN0aW9uKGUpe3JldHVybiFpLndpbGxMaW5rQWN0aXZhdGV8fGkud2lsbExpbmtBY3RpdmF0ZShlLHIpfSkpKX19LGUucHJvdG90eXBlLl9jcmVhdGVMaW5rSG92ZXJFdmVudD1mdW5jdGlvbihlLHQscixpLG4pe3JldHVybnt4MTplLHkxOnQseDI6cix5MjppLGNvbHM6dGhpcy5fYnVmZmVyU2VydmljZS5jb2xzLGZnOm59fSxlLl90aW1lQmVmb3JlTGF0ZW5jeT0yMDAsZT1pKFtuKDAscy5JQnVmZmVyU2VydmljZSksbigxLHMuSUxvZ1NlcnZpY2UpLG4oMixzLklVbmljb2RlU2VydmljZSldLGUpfSgpO3QuTGlua2lmaWVyPWE7dmFyIGM9ZnVuY3Rpb24oZSx0LHIsaSxuLG8scyxhLGMpe3RoaXMueDE9ZSx0aGlzLnkxPXQsdGhpcy54Mj1yLHRoaXMueTI9aSx0aGlzLmNsaWNrQ2FsbGJhY2s9bix0aGlzLmhvdmVyQ2FsbGJhY2s9byx0aGlzLnRvb2x0aXBDYWxsYmFjaz1zLHRoaXMubGVhdmVDYWxsYmFjaz1hLHRoaXMud2lsbExpbmtBY3RpdmF0ZT1jfTt0Lk1vdXNlWm9uZT1jfSw2NDY1OmZ1bmN0aW9uKGUsdCxyKXt2YXIgaSxuPXRoaXMmJnRoaXMuX19leHRlbmRzfHwoaT1mdW5jdGlvbihlLHQpe3JldHVybiBpPU9iamVjdC5zZXRQcm90b3R5cGVPZnx8e19fcHJvdG9fXzpbXX1pbnN0YW5jZW9mIEFycmF5JiZmdW5jdGlvbihlLHQpe2UuX19wcm90b19fPXR9fHxmdW5jdGlvbihlLHQpe2Zvcih2YXIgciBpbiB0KU9iamVjdC5wcm90b3R5cGUuaGFzT3duUHJvcGVydHkuY2FsbCh0LHIpJiYoZVtyXT10W3JdKX0saShlLHQpfSxmdW5jdGlvbihlLHQpe2lmKCJmdW5jdGlvbiIhPXR5cGVvZiB0JiZudWxsIT09dCl0aHJvdyBuZXcgVHlwZUVycm9yKCJDbGFzcyBleHRlbmRzIHZhbHVlICIrU3RyaW5nKHQpKyIgaXMgbm90IGEgY29uc3RydWN0b3Igb3IgbnVsbCIpO2Z1bmN0aW9uIHIoKXt0aGlzLmNvbnN0cnVjdG9yPWV9aShlLHQpLGUucHJvdG90eXBlPW51bGw9PT10P09iamVjdC5jcmVhdGUodCk6KHIucHJvdG90eXBlPXQucHJvdG90eXBlLG5ldyByKX0pLG89dGhpcyYmdGhpcy5fX2RlY29yYXRlfHxmdW5jdGlvbihlLHQscixpKXt2YXIgbixvPWFyZ3VtZW50cy5sZW5ndGgscz1vPDM/dDpudWxsPT09aT9pPU9iamVjdC5nZXRPd25Qcm9wZXJ0eURlc2NyaXB0b3IodCxyKTppO2lmKCJvYmplY3QiPT10eXBlb2YgUmVmbGVjdCYmImZ1bmN0aW9uIj09dHlwZW9mIFJlZmxlY3QuZGVjb3JhdGUpcz1SZWZsZWN0LmRlY29yYXRlKGUsdCxyLGkpO2Vsc2UgZm9yKHZhciBhPWUubGVuZ3RoLTE7YT49MDthLS0pKG49ZVthXSkmJihzPShvPDM/bihzKTpvPjM/bih0LHIscyk6bih0LHIpKXx8cyk7cmV0dXJuIG8+MyYmcyYmT2JqZWN0LmRlZmluZVByb3BlcnR5KHQscixzKSxzfSxzPXRoaXMmJnRoaXMuX19wYXJhbXx8ZnVuY3Rpb24oZSx0KXtyZXR1cm4gZnVuY3Rpb24ocixpKXt0KHIsaSxlKX19O09iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LCJfX2VzTW9kdWxlIix7dmFsdWU6ITB9KSx0LkxpbmtpZmllcjI9dm9pZCAwO3ZhciBhPXIoMjU4NSksYz1yKDg0NjApLGw9cig4NDQpLHU9cigzNjU2KSxoPWZ1bmN0aW9uKGUpe2Z1bmN0aW9uIHQodCl7dmFyIHI9ZS5jYWxsKHRoaXMpfHx0aGlzO3JldHVybiByLl9idWZmZXJTZXJ2aWNlPXQsci5fbGlua1Byb3ZpZGVycz1bXSxyLl9saW5rQ2FjaGVEaXNwb3NhYmxlcz1bXSxyLl9pc01vdXNlT3V0PSEwLHIuX2FjdGl2ZUxpbmU9LTEsci5fb25TaG93TGlua1VuZGVybGluZT1yLnJlZ2lzdGVyKG5ldyBjLkV2ZW50RW1pdHRlciksci5fb25IaWRlTGlua1VuZGVybGluZT1yLnJlZ2lzdGVyKG5ldyBjLkV2ZW50RW1pdHRlciksci5yZWdpc3RlcigoMCxsLmdldERpc3Bvc2VBcnJheURpc3Bvc2FibGUpKHIuX2xpbmtDYWNoZURpc3Bvc2FibGVzKSkscn1yZXR1cm4gbih0LGUpLE9iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LnByb3RvdHlwZSwiY3VycmVudExpbmsiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fY3VycmVudExpbmt9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KHQucHJvdG90eXBlLCJvblNob3dMaW5rVW5kZXJsaW5lIix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX29uU2hvd0xpbmtVbmRlcmxpbmUuZXZlbnR9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KHQucHJvdG90eXBlLCJvbkhpZGVMaW5rVW5kZXJsaW5lIix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX29uSGlkZUxpbmtVbmRlcmxpbmUuZXZlbnR9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksdC5wcm90b3R5cGUucmVnaXN0ZXJMaW5rUHJvdmlkZXI9ZnVuY3Rpb24oZSl7dmFyIHQ9dGhpcztyZXR1cm4gdGhpcy5fbGlua1Byb3ZpZGVycy5wdXNoKGUpLHtkaXNwb3NlOmZ1bmN0aW9uKCl7dmFyIHI9dC5fbGlua1Byb3ZpZGVycy5pbmRleE9mKGUpOy0xIT09ciYmdC5fbGlua1Byb3ZpZGVycy5zcGxpY2UociwxKX19fSx0LnByb3RvdHlwZS5hdHRhY2hUb0RvbT1mdW5jdGlvbihlLHQscil7dmFyIGk9dGhpczt0aGlzLl9lbGVtZW50PWUsdGhpcy5fbW91c2VTZXJ2aWNlPXQsdGhpcy5fcmVuZGVyU2VydmljZT1yLHRoaXMucmVnaXN0ZXIoKDAsdS5hZGREaXNwb3NhYmxlRG9tTGlzdGVuZXIpKHRoaXMuX2VsZW1lbnQsIm1vdXNlbGVhdmUiLChmdW5jdGlvbigpe2kuX2lzTW91c2VPdXQ9ITAsaS5fY2xlYXJDdXJyZW50TGluaygpfSkpKSx0aGlzLnJlZ2lzdGVyKCgwLHUuYWRkRGlzcG9zYWJsZURvbUxpc3RlbmVyKSh0aGlzLl9lbGVtZW50LCJtb3VzZW1vdmUiLHRoaXMuX29uTW91c2VNb3ZlLmJpbmQodGhpcykpKSx0aGlzLnJlZ2lzdGVyKCgwLHUuYWRkRGlzcG9zYWJsZURvbUxpc3RlbmVyKSh0aGlzLl9lbGVtZW50LCJjbGljayIsdGhpcy5fb25DbGljay5iaW5kKHRoaXMpKSl9LHQucHJvdG90eXBlLl9vbk1vdXNlTW92ZT1mdW5jdGlvbihlKXtpZih0aGlzLl9sYXN0TW91c2VFdmVudD1lLHRoaXMuX2VsZW1lbnQmJnRoaXMuX21vdXNlU2VydmljZSl7dmFyIHQ9dGhpcy5fcG9zaXRpb25Gcm9tTW91c2VFdmVudChlLHRoaXMuX2VsZW1lbnQsdGhpcy5fbW91c2VTZXJ2aWNlKTtpZih0KXt0aGlzLl9pc01vdXNlT3V0PSExO2Zvcih2YXIgcj1lLmNvbXBvc2VkUGF0aCgpLGk9MDtpPHIubGVuZ3RoO2krKyl7dmFyIG49cltpXTtpZihuLmNsYXNzTGlzdC5jb250YWlucygieHRlcm0iKSlicmVhaztpZihuLmNsYXNzTGlzdC5jb250YWlucygieHRlcm0taG92ZXIiKSlyZXR1cm59dGhpcy5fbGFzdEJ1ZmZlckNlbGwmJnQueD09PXRoaXMuX2xhc3RCdWZmZXJDZWxsLngmJnQueT09PXRoaXMuX2xhc3RCdWZmZXJDZWxsLnl8fCh0aGlzLl9vbkhvdmVyKHQpLHRoaXMuX2xhc3RCdWZmZXJDZWxsPXQpfX19LHQucHJvdG90eXBlLl9vbkhvdmVyPWZ1bmN0aW9uKGUpe2lmKHRoaXMuX2FjdGl2ZUxpbmUhPT1lLnkpcmV0dXJuIHRoaXMuX2NsZWFyQ3VycmVudExpbmsoKSx2b2lkIHRoaXMuX2Fza0ZvckxpbmsoZSwhMSk7dGhpcy5fY3VycmVudExpbmsmJnRoaXMuX2xpbmtBdFBvc2l0aW9uKHRoaXMuX2N1cnJlbnRMaW5rLmxpbmssZSl8fCh0aGlzLl9jbGVhckN1cnJlbnRMaW5rKCksdGhpcy5fYXNrRm9yTGluayhlLCEwKSl9LHQucHJvdG90eXBlLl9hc2tGb3JMaW5rPWZ1bmN0aW9uKGUsdCl7dmFyIHIsaT10aGlzO3RoaXMuX2FjdGl2ZVByb3ZpZGVyUmVwbGllcyYmdHx8KG51bGw9PT0ocj10aGlzLl9hY3RpdmVQcm92aWRlclJlcGxpZXMpfHx2b2lkIDA9PT1yfHxyLmZvckVhY2goKGZ1bmN0aW9uKGUpe251bGw9PWV8fGUuZm9yRWFjaCgoZnVuY3Rpb24oZSl7ZS5saW5rLmRpc3Bvc2UmJmUubGluay5kaXNwb3NlKCl9KSl9KSksdGhpcy5fYWN0aXZlUHJvdmlkZXJSZXBsaWVzPW5ldyBNYXAsdGhpcy5fYWN0aXZlTGluZT1lLnkpO3ZhciBuPSExO3RoaXMuX2xpbmtQcm92aWRlcnMuZm9yRWFjaCgoZnVuY3Rpb24ocixvKXt2YXIgczt0PyhudWxsPT09KHM9aS5fYWN0aXZlUHJvdmlkZXJSZXBsaWVzKXx8dm9pZCAwPT09cz92b2lkIDA6cy5nZXQobykpJiYobj1pLl9jaGVja0xpbmtQcm92aWRlclJlc3VsdChvLGUsbikpOnIucHJvdmlkZUxpbmtzKGUueSwoZnVuY3Rpb24odCl7dmFyIHIscztpZighaS5faXNNb3VzZU91dCl7dmFyIGE9bnVsbD09dD92b2lkIDA6dC5tYXAoKGZ1bmN0aW9uKGUpe3JldHVybntsaW5rOmV9fSkpO251bGw9PT0ocj1pLl9hY3RpdmVQcm92aWRlclJlcGxpZXMpfHx2b2lkIDA9PT1yfHxyLnNldChvLGEpLG49aS5fY2hlY2tMaW5rUHJvdmlkZXJSZXN1bHQobyxlLG4pLChudWxsPT09KHM9aS5fYWN0aXZlUHJvdmlkZXJSZXBsaWVzKXx8dm9pZCAwPT09cz92b2lkIDA6cy5zaXplKT09PWkuX2xpbmtQcm92aWRlcnMubGVuZ3RoJiZpLl9yZW1vdmVJbnRlcnNlY3RpbmdMaW5rcyhlLnksaS5fYWN0aXZlUHJvdmlkZXJSZXBsaWVzKX19KSl9KSl9LHQucHJvdG90eXBlLl9yZW1vdmVJbnRlcnNlY3RpbmdMaW5rcz1mdW5jdGlvbihlLHQpe2Zvcih2YXIgcj1uZXcgU2V0LGk9MDtpPHQuc2l6ZTtpKyspe3ZhciBuPXQuZ2V0KGkpO2lmKG4pZm9yKHZhciBvPTA7bzxuLmxlbmd0aDtvKyspZm9yKHZhciBzPW5bb10sYT1zLmxpbmsucmFuZ2Uuc3RhcnQueTxlPzA6cy5saW5rLnJhbmdlLnN0YXJ0LngsYz1zLmxpbmsucmFuZ2UuZW5kLnk+ZT90aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHM6cy5saW5rLnJhbmdlLmVuZC54LGw9YTtsPD1jO2wrKyl7aWYoci5oYXMobCkpe24uc3BsaWNlKG8tLSwxKTticmVha31yLmFkZChsKX19fSx0LnByb3RvdHlwZS5fY2hlY2tMaW5rUHJvdmlkZXJSZXN1bHQ9ZnVuY3Rpb24oZSx0LHIpe3ZhciBpLG49dGhpcztpZighdGhpcy5fYWN0aXZlUHJvdmlkZXJSZXBsaWVzKXJldHVybiByO2Zvcih2YXIgbz10aGlzLl9hY3RpdmVQcm92aWRlclJlcGxpZXMuZ2V0KGUpLHM9ITEsYT0wO2E8ZTthKyspdGhpcy5fYWN0aXZlUHJvdmlkZXJSZXBsaWVzLmhhcyhhKSYmIXRoaXMuX2FjdGl2ZVByb3ZpZGVyUmVwbGllcy5nZXQoYSl8fChzPSEwKTtpZighcyYmbyl7dmFyIGM9by5maW5kKChmdW5jdGlvbihlKXtyZXR1cm4gbi5fbGlua0F0UG9zaXRpb24oZS5saW5rLHQpfSkpO2MmJihyPSEwLHRoaXMuX2hhbmRsZU5ld0xpbmsoYykpfWlmKHRoaXMuX2FjdGl2ZVByb3ZpZGVyUmVwbGllcy5zaXplPT09dGhpcy5fbGlua1Byb3ZpZGVycy5sZW5ndGgmJiFyKWZvcihhPTA7YTx0aGlzLl9hY3RpdmVQcm92aWRlclJlcGxpZXMuc2l6ZTthKyspe3ZhciBsPW51bGw9PT0oaT10aGlzLl9hY3RpdmVQcm92aWRlclJlcGxpZXMuZ2V0KGEpKXx8dm9pZCAwPT09aT92b2lkIDA6aS5maW5kKChmdW5jdGlvbihlKXtyZXR1cm4gbi5fbGlua0F0UG9zaXRpb24oZS5saW5rLHQpfSkpO2lmKGwpe3I9ITAsdGhpcy5faGFuZGxlTmV3TGluayhsKTticmVha319cmV0dXJuIHJ9LHQucHJvdG90eXBlLl9vbkNsaWNrPWZ1bmN0aW9uKGUpe2lmKHRoaXMuX2VsZW1lbnQmJnRoaXMuX21vdXNlU2VydmljZSYmdGhpcy5fY3VycmVudExpbmspe3ZhciB0PXRoaXMuX3Bvc2l0aW9uRnJvbU1vdXNlRXZlbnQoZSx0aGlzLl9lbGVtZW50LHRoaXMuX21vdXNlU2VydmljZSk7dCYmdGhpcy5fbGlua0F0UG9zaXRpb24odGhpcy5fY3VycmVudExpbmsubGluayx0KSYmdGhpcy5fY3VycmVudExpbmsubGluay5hY3RpdmF0ZShlLHRoaXMuX2N1cnJlbnRMaW5rLmxpbmsudGV4dCl9fSx0LnByb3RvdHlwZS5fY2xlYXJDdXJyZW50TGluaz1mdW5jdGlvbihlLHQpe3RoaXMuX2VsZW1lbnQmJnRoaXMuX2N1cnJlbnRMaW5rJiZ0aGlzLl9sYXN0TW91c2VFdmVudCYmKCFlfHwhdHx8dGhpcy5fY3VycmVudExpbmsubGluay5yYW5nZS5zdGFydC55Pj1lJiZ0aGlzLl9jdXJyZW50TGluay5saW5rLnJhbmdlLmVuZC55PD10KSYmKHRoaXMuX2xpbmtMZWF2ZSh0aGlzLl9lbGVtZW50LHRoaXMuX2N1cnJlbnRMaW5rLmxpbmssdGhpcy5fbGFzdE1vdXNlRXZlbnQpLHRoaXMuX2N1cnJlbnRMaW5rPXZvaWQgMCwoMCxsLmRpc3Bvc2VBcnJheSkodGhpcy5fbGlua0NhY2hlRGlzcG9zYWJsZXMpKX0sdC5wcm90b3R5cGUuX2hhbmRsZU5ld0xpbms9ZnVuY3Rpb24oZSl7dmFyIHQ9dGhpcztpZih0aGlzLl9lbGVtZW50JiZ0aGlzLl9sYXN0TW91c2VFdmVudCYmdGhpcy5fbW91c2VTZXJ2aWNlKXt2YXIgcj10aGlzLl9wb3NpdGlvbkZyb21Nb3VzZUV2ZW50KHRoaXMuX2xhc3RNb3VzZUV2ZW50LHRoaXMuX2VsZW1lbnQsdGhpcy5fbW91c2VTZXJ2aWNlKTtyJiZ0aGlzLl9saW5rQXRQb3NpdGlvbihlLmxpbmsscikmJih0aGlzLl9jdXJyZW50TGluaz1lLHRoaXMuX2N1cnJlbnRMaW5rLnN0YXRlPXtkZWNvcmF0aW9uczp7dW5kZXJsaW5lOnZvaWQgMD09PWUubGluay5kZWNvcmF0aW9uc3x8ZS5saW5rLmRlY29yYXRpb25zLnVuZGVybGluZSxwb2ludGVyQ3Vyc29yOnZvaWQgMD09PWUubGluay5kZWNvcmF0aW9uc3x8ZS5saW5rLmRlY29yYXRpb25zLnBvaW50ZXJDdXJzb3J9LGlzSG92ZXJlZDohMH0sdGhpcy5fbGlua0hvdmVyKHRoaXMuX2VsZW1lbnQsZS5saW5rLHRoaXMuX2xhc3RNb3VzZUV2ZW50KSxlLmxpbmsuZGVjb3JhdGlvbnM9e30sT2JqZWN0LmRlZmluZVByb3BlcnRpZXMoZS5saW5rLmRlY29yYXRpb25zLHtwb2ludGVyQ3Vyc29yOntnZXQ6ZnVuY3Rpb24oKXt2YXIgZSxyO3JldHVybiBudWxsPT09KHI9bnVsbD09PShlPXQuX2N1cnJlbnRMaW5rKXx8dm9pZCAwPT09ZT92b2lkIDA6ZS5zdGF0ZSl8fHZvaWQgMD09PXI/dm9pZCAwOnIuZGVjb3JhdGlvbnMucG9pbnRlckN1cnNvcn0sc2V0OmZ1bmN0aW9uKGUpe3ZhciByLGk7KG51bGw9PT0ocj10Ll9jdXJyZW50TGluayl8fHZvaWQgMD09PXI/dm9pZCAwOnIuc3RhdGUpJiZ0Ll9jdXJyZW50TGluay5zdGF0ZS5kZWNvcmF0aW9ucy5wb2ludGVyQ3Vyc29yIT09ZSYmKHQuX2N1cnJlbnRMaW5rLnN0YXRlLmRlY29yYXRpb25zLnBvaW50ZXJDdXJzb3I9ZSx0Ll9jdXJyZW50TGluay5zdGF0ZS5pc0hvdmVyZWQmJihudWxsPT09KGk9dC5fZWxlbWVudCl8fHZvaWQgMD09PWl8fGkuY2xhc3NMaXN0LnRvZ2dsZSgieHRlcm0tY3Vyc29yLXBvaW50ZXIiLGUpKSl9fSx1bmRlcmxpbmU6e2dldDpmdW5jdGlvbigpe3ZhciBlLHI7cmV0dXJuIG51bGw9PT0ocj1udWxsPT09KGU9dC5fY3VycmVudExpbmspfHx2b2lkIDA9PT1lP3ZvaWQgMDplLnN0YXRlKXx8dm9pZCAwPT09cj92b2lkIDA6ci5kZWNvcmF0aW9ucy51bmRlcmxpbmV9LHNldDpmdW5jdGlvbihyKXt2YXIgaSxuLG87KG51bGw9PT0oaT10Ll9jdXJyZW50TGluayl8fHZvaWQgMD09PWk/dm9pZCAwOmkuc3RhdGUpJiYobnVsbD09PShvPW51bGw9PT0obj10Ll9jdXJyZW50TGluayl8fHZvaWQgMD09PW4/dm9pZCAwOm4uc3RhdGUpfHx2b2lkIDA9PT1vP3ZvaWQgMDpvLmRlY29yYXRpb25zLnVuZGVybGluZSkhPT1yJiYodC5fY3VycmVudExpbmsuc3RhdGUuZGVjb3JhdGlvbnMudW5kZXJsaW5lPXIsdC5fY3VycmVudExpbmsuc3RhdGUuaXNIb3ZlcmVkJiZ0Ll9maXJlVW5kZXJsaW5lRXZlbnQoZS5saW5rLHIpKX19fSksdGhpcy5fcmVuZGVyU2VydmljZSYmdGhpcy5fbGlua0NhY2hlRGlzcG9zYWJsZXMucHVzaCh0aGlzLl9yZW5kZXJTZXJ2aWNlLm9uUmVuZGVyZWRCdWZmZXJDaGFuZ2UoKGZ1bmN0aW9uKGUpe3ZhciByPTA9PT1lLnN0YXJ0PzA6ZS5zdGFydCsxK3QuX2J1ZmZlclNlcnZpY2UuYnVmZmVyLnlkaXNwO3QuX2NsZWFyQ3VycmVudExpbmsocixlLmVuZCsxK3QuX2J1ZmZlclNlcnZpY2UuYnVmZmVyLnlkaXNwKX0pKSkpfX0sdC5wcm90b3R5cGUuX2xpbmtIb3Zlcj1mdW5jdGlvbihlLHQscil7dmFyIGk7KG51bGw9PT0oaT10aGlzLl9jdXJyZW50TGluayl8fHZvaWQgMD09PWk/dm9pZCAwOmkuc3RhdGUpJiYodGhpcy5fY3VycmVudExpbmsuc3RhdGUuaXNIb3ZlcmVkPSEwLHRoaXMuX2N1cnJlbnRMaW5rLnN0YXRlLmRlY29yYXRpb25zLnVuZGVybGluZSYmdGhpcy5fZmlyZVVuZGVybGluZUV2ZW50KHQsITApLHRoaXMuX2N1cnJlbnRMaW5rLnN0YXRlLmRlY29yYXRpb25zLnBvaW50ZXJDdXJzb3ImJmUuY2xhc3NMaXN0LmFkZCgieHRlcm0tY3Vyc29yLXBvaW50ZXIiKSksdC5ob3ZlciYmdC5ob3ZlcihyLHQudGV4dCl9LHQucHJvdG90eXBlLl9maXJlVW5kZXJsaW5lRXZlbnQ9ZnVuY3Rpb24oZSx0KXt2YXIgcj1lLnJhbmdlLGk9dGhpcy5fYnVmZmVyU2VydmljZS5idWZmZXIueWRpc3Asbj10aGlzLl9jcmVhdGVMaW5rVW5kZXJsaW5lRXZlbnQoci5zdGFydC54LTEsci5zdGFydC55LWktMSxyLmVuZC54LHIuZW5kLnktaS0xLHZvaWQgMCk7KHQ/dGhpcy5fb25TaG93TGlua1VuZGVybGluZTp0aGlzLl9vbkhpZGVMaW5rVW5kZXJsaW5lKS5maXJlKG4pfSx0LnByb3RvdHlwZS5fbGlua0xlYXZlPWZ1bmN0aW9uKGUsdCxyKXt2YXIgaTsobnVsbD09PShpPXRoaXMuX2N1cnJlbnRMaW5rKXx8dm9pZCAwPT09aT92b2lkIDA6aS5zdGF0ZSkmJih0aGlzLl9jdXJyZW50TGluay5zdGF0ZS5pc0hvdmVyZWQ9ITEsdGhpcy5fY3VycmVudExpbmsuc3RhdGUuZGVjb3JhdGlvbnMudW5kZXJsaW5lJiZ0aGlzLl9maXJlVW5kZXJsaW5lRXZlbnQodCwhMSksdGhpcy5fY3VycmVudExpbmsuc3RhdGUuZGVjb3JhdGlvbnMucG9pbnRlckN1cnNvciYmZS5jbGFzc0xpc3QucmVtb3ZlKCJ4dGVybS1jdXJzb3ItcG9pbnRlciIpKSx0LmxlYXZlJiZ0LmxlYXZlKHIsdC50ZXh0KX0sdC5wcm90b3R5cGUuX2xpbmtBdFBvc2l0aW9uPWZ1bmN0aW9uKGUsdCl7dmFyIHI9ZS5yYW5nZS5zdGFydC55PT09ZS5yYW5nZS5lbmQueSxpPWUucmFuZ2Uuc3RhcnQueTx0Lnksbj1lLnJhbmdlLmVuZC55PnQueTtyZXR1cm4ociYmZS5yYW5nZS5zdGFydC54PD10LngmJmUucmFuZ2UuZW5kLng+PXQueHx8aSYmZS5yYW5nZS5lbmQueD49dC54fHxuJiZlLnJhbmdlLnN0YXJ0Lng8PXQueHx8aSYmbikmJmUucmFuZ2Uuc3RhcnQueTw9dC55JiZlLnJhbmdlLmVuZC55Pj10Lnl9LHQucHJvdG90eXBlLl9wb3NpdGlvbkZyb21Nb3VzZUV2ZW50PWZ1bmN0aW9uKGUsdCxyKXt2YXIgaT1yLmdldENvb3JkcyhlLHQsdGhpcy5fYnVmZmVyU2VydmljZS5jb2xzLHRoaXMuX2J1ZmZlclNlcnZpY2Uucm93cyk7aWYoaSlyZXR1cm57eDppWzBdLHk6aVsxXSt0aGlzLl9idWZmZXJTZXJ2aWNlLmJ1ZmZlci55ZGlzcH19LHQucHJvdG90eXBlLl9jcmVhdGVMaW5rVW5kZXJsaW5lRXZlbnQ9ZnVuY3Rpb24oZSx0LHIsaSxuKXtyZXR1cm57eDE6ZSx5MTp0LHgyOnIseTI6aSxjb2xzOnRoaXMuX2J1ZmZlclNlcnZpY2UuY29scyxmZzpufX0sbyhbcygwLGEuSUJ1ZmZlclNlcnZpY2UpXSx0KX0obC5EaXNwb3NhYmxlKTt0LkxpbmtpZmllcjI9aH0sOTA0MjooZSx0KT0+e09iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LCJfX2VzTW9kdWxlIix7dmFsdWU6ITB9KSx0LnRvb011Y2hPdXRwdXQ9dC5wcm9tcHRMYWJlbD12b2lkIDAsdC5wcm9tcHRMYWJlbD0iVGVybWluYWwgaW5wdXQiLHQudG9vTXVjaE91dHB1dD0iVG9vIG11Y2ggb3V0cHV0IHRvIGFubm91bmNlLCBuYXZpZ2F0ZSB0byByb3dzIG1hbnVhbGx5IHRvIHJlYWQifSw2OTU0OmZ1bmN0aW9uKGUsdCxyKXt2YXIgaSxuPXRoaXMmJnRoaXMuX19leHRlbmRzfHwoaT1mdW5jdGlvbihlLHQpe3JldHVybiBpPU9iamVjdC5zZXRQcm90b3R5cGVPZnx8e19fcHJvdG9fXzpbXX1pbnN0YW5jZW9mIEFycmF5JiZmdW5jdGlvbihlLHQpe2UuX19wcm90b19fPXR9fHxmdW5jdGlvbihlLHQpe2Zvcih2YXIgciBpbiB0KU9iamVjdC5wcm90b3R5cGUuaGFzT3duUHJvcGVydHkuY2FsbCh0LHIpJiYoZVtyXT10W3JdKX0saShlLHQpfSxmdW5jdGlvbihlLHQpe2lmKCJmdW5jdGlvbiIhPXR5cGVvZiB0JiZudWxsIT09dCl0aHJvdyBuZXcgVHlwZUVycm9yKCJDbGFzcyBleHRlbmRzIHZhbHVlICIrU3RyaW5nKHQpKyIgaXMgbm90IGEgY29uc3RydWN0b3Igb3IgbnVsbCIpO2Z1bmN0aW9uIHIoKXt0aGlzLmNvbnN0cnVjdG9yPWV9aShlLHQpLGUucHJvdG90eXBlPW51bGw9PT10P09iamVjdC5jcmVhdGUodCk6KHIucHJvdG90eXBlPXQucHJvdG90eXBlLG5ldyByKX0pLG89dGhpcyYmdGhpcy5fX2RlY29yYXRlfHxmdW5jdGlvbihlLHQscixpKXt2YXIgbixvPWFyZ3VtZW50cy5sZW5ndGgscz1vPDM/dDpudWxsPT09aT9pPU9iamVjdC5nZXRPd25Qcm9wZXJ0eURlc2NyaXB0b3IodCxyKTppO2lmKCJvYmplY3QiPT10eXBlb2YgUmVmbGVjdCYmImZ1bmN0aW9uIj09dHlwZW9mIFJlZmxlY3QuZGVjb3JhdGUpcz1SZWZsZWN0LmRlY29yYXRlKGUsdCxyLGkpO2Vsc2UgZm9yKHZhciBhPWUubGVuZ3RoLTE7YT49MDthLS0pKG49ZVthXSkmJihzPShvPDM/bihzKTpvPjM/bih0LHIscyk6bih0LHIpKXx8cyk7cmV0dXJuIG8+MyYmcyYmT2JqZWN0LmRlZmluZVByb3BlcnR5KHQscixzKSxzfSxzPXRoaXMmJnRoaXMuX19wYXJhbXx8ZnVuY3Rpb24oZSx0KXtyZXR1cm4gZnVuY3Rpb24ocixpKXt0KHIsaSxlKX19O09iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LCJfX2VzTW9kdWxlIix7dmFsdWU6ITB9KSx0Lk1vdXNlWm9uZU1hbmFnZXI9dm9pZCAwO3ZhciBhPXIoODQ0KSxjPXIoMzY1NiksbD1yKDQ3MjUpLHU9cigyNTg1KSxoPWZ1bmN0aW9uKGUpe2Z1bmN0aW9uIHQodCxyLGksbixvLHMpe3ZhciBhPWUuY2FsbCh0aGlzKXx8dGhpcztyZXR1cm4gYS5fZWxlbWVudD10LGEuX3NjcmVlbkVsZW1lbnQ9cixhLl9idWZmZXJTZXJ2aWNlPWksYS5fbW91c2VTZXJ2aWNlPW4sYS5fc2VsZWN0aW9uU2VydmljZT1vLGEuX29wdGlvbnNTZXJ2aWNlPXMsYS5fem9uZXM9W10sYS5fYXJlWm9uZXNBY3RpdmU9ITEsYS5fbGFzdEhvdmVyQ29vcmRzPVt2b2lkIDAsdm9pZCAwXSxhLl9pbml0aWFsU2VsZWN0aW9uTGVuZ3RoPTAsYS5yZWdpc3RlcigoMCxjLmFkZERpc3Bvc2FibGVEb21MaXN0ZW5lcikoYS5fZWxlbWVudCwibW91c2Vkb3duIiwoZnVuY3Rpb24oZSl7cmV0dXJuIGEuX29uTW91c2VEb3duKGUpfSkpKSxhLl9tb3VzZU1vdmVMaXN0ZW5lcj1mdW5jdGlvbihlKXtyZXR1cm4gYS5fb25Nb3VzZU1vdmUoZSl9LGEuX21vdXNlTGVhdmVMaXN0ZW5lcj1mdW5jdGlvbihlKXtyZXR1cm4gYS5fb25Nb3VzZUxlYXZlKGUpfSxhLl9jbGlja0xpc3RlbmVyPWZ1bmN0aW9uKGUpe3JldHVybiBhLl9vbkNsaWNrKGUpfSxhfXJldHVybiBuKHQsZSksdC5wcm90b3R5cGUuZGlzcG9zZT1mdW5jdGlvbigpe2UucHJvdG90eXBlLmRpc3Bvc2UuY2FsbCh0aGlzKSx0aGlzLl9kZWFjdGl2YXRlKCl9LHQucHJvdG90eXBlLmFkZD1mdW5jdGlvbihlKXt0aGlzLl96b25lcy5wdXNoKGUpLDE9PT10aGlzLl96b25lcy5sZW5ndGgmJnRoaXMuX2FjdGl2YXRlKCl9LHQucHJvdG90eXBlLmNsZWFyQWxsPWZ1bmN0aW9uKGUsdCl7aWYoMCE9PXRoaXMuX3pvbmVzLmxlbmd0aCl7ZSYmdHx8KGU9MCx0PXRoaXMuX2J1ZmZlclNlcnZpY2Uucm93cy0xKTtmb3IodmFyIHI9MDtyPHRoaXMuX3pvbmVzLmxlbmd0aDtyKyspe3ZhciBpPXRoaXMuX3pvbmVzW3JdOyhpLnkxPmUmJmkueTE8PXQrMXx8aS55Mj5lJiZpLnkyPD10KzF8fGkueTE8ZSYmaS55Mj50KzEpJiYodGhpcy5fY3VycmVudFpvbmUmJnRoaXMuX2N1cnJlbnRab25lPT09aSYmKHRoaXMuX2N1cnJlbnRab25lLmxlYXZlQ2FsbGJhY2soKSx0aGlzLl9jdXJyZW50Wm9uZT12b2lkIDApLHRoaXMuX3pvbmVzLnNwbGljZShyLS0sMSkpfTA9PT10aGlzLl96b25lcy5sZW5ndGgmJnRoaXMuX2RlYWN0aXZhdGUoKX19LHQucHJvdG90eXBlLl9hY3RpdmF0ZT1mdW5jdGlvbigpe3RoaXMuX2FyZVpvbmVzQWN0aXZlfHwodGhpcy5fYXJlWm9uZXNBY3RpdmU9ITAsdGhpcy5fZWxlbWVudC5hZGRFdmVudExpc3RlbmVyKCJtb3VzZW1vdmUiLHRoaXMuX21vdXNlTW92ZUxpc3RlbmVyKSx0aGlzLl9lbGVtZW50LmFkZEV2ZW50TGlzdGVuZXIoIm1vdXNlbGVhdmUiLHRoaXMuX21vdXNlTGVhdmVMaXN0ZW5lciksdGhpcy5fZWxlbWVudC5hZGRFdmVudExpc3RlbmVyKCJjbGljayIsdGhpcy5fY2xpY2tMaXN0ZW5lcikpfSx0LnByb3RvdHlwZS5fZGVhY3RpdmF0ZT1mdW5jdGlvbigpe3RoaXMuX2FyZVpvbmVzQWN0aXZlJiYodGhpcy5fYXJlWm9uZXNBY3RpdmU9ITEsdGhpcy5fZWxlbWVudC5yZW1vdmVFdmVudExpc3RlbmVyKCJtb3VzZW1vdmUiLHRoaXMuX21vdXNlTW92ZUxpc3RlbmVyKSx0aGlzLl9lbGVtZW50LnJlbW92ZUV2ZW50TGlzdGVuZXIoIm1vdXNlbGVhdmUiLHRoaXMuX21vdXNlTGVhdmVMaXN0ZW5lciksdGhpcy5fZWxlbWVudC5yZW1vdmVFdmVudExpc3RlbmVyKCJjbGljayIsdGhpcy5fY2xpY2tMaXN0ZW5lcikpfSx0LnByb3RvdHlwZS5fb25Nb3VzZU1vdmU9ZnVuY3Rpb24oZSl7dGhpcy5fbGFzdEhvdmVyQ29vcmRzWzBdPT09ZS5wYWdlWCYmdGhpcy5fbGFzdEhvdmVyQ29vcmRzWzFdPT09ZS5wYWdlWXx8KHRoaXMuX29uSG92ZXIoZSksdGhpcy5fbGFzdEhvdmVyQ29vcmRzPVtlLnBhZ2VYLGUucGFnZVldKX0sdC5wcm90b3R5cGUuX29uSG92ZXI9ZnVuY3Rpb24oZSl7dmFyIHQ9dGhpcyxyPXRoaXMuX2ZpbmRab25lRXZlbnRBdChlKTtyIT09dGhpcy5fY3VycmVudFpvbmUmJih0aGlzLl9jdXJyZW50Wm9uZSYmKHRoaXMuX2N1cnJlbnRab25lLmxlYXZlQ2FsbGJhY2soKSx0aGlzLl9jdXJyZW50Wm9uZT12b2lkIDAsdGhpcy5fdG9vbHRpcFRpbWVvdXQmJmNsZWFyVGltZW91dCh0aGlzLl90b29sdGlwVGltZW91dCkpLHImJih0aGlzLl9jdXJyZW50Wm9uZT1yLHIuaG92ZXJDYWxsYmFjayYmci5ob3ZlckNhbGxiYWNrKGUpLHRoaXMuX3Rvb2x0aXBUaW1lb3V0PXdpbmRvdy5zZXRUaW1lb3V0KChmdW5jdGlvbigpe3JldHVybiB0Ll9vblRvb2x0aXAoZSl9KSx0aGlzLl9vcHRpb25zU2VydmljZS5vcHRpb25zLmxpbmtUb29sdGlwSG92ZXJEdXJhdGlvbikpKX0sdC5wcm90b3R5cGUuX29uVG9vbHRpcD1mdW5jdGlvbihlKXt0aGlzLl90b29sdGlwVGltZW91dD12b2lkIDA7dmFyIHQ9dGhpcy5fZmluZFpvbmVFdmVudEF0KGUpO251bGw9PXR8fHQudG9vbHRpcENhbGxiYWNrKGUpfSx0LnByb3RvdHlwZS5fb25Nb3VzZURvd249ZnVuY3Rpb24oZSl7aWYodGhpcy5faW5pdGlhbFNlbGVjdGlvbkxlbmd0aD10aGlzLl9nZXRTZWxlY3Rpb25MZW5ndGgoKSx0aGlzLl9hcmVab25lc0FjdGl2ZSl7dmFyIHQ9dGhpcy5fZmluZFpvbmVFdmVudEF0KGUpOyhudWxsPT10P3ZvaWQgMDp0LndpbGxMaW5rQWN0aXZhdGUoZSkpJiYoZS5wcmV2ZW50RGVmYXVsdCgpLGUuc3RvcEltbWVkaWF0ZVByb3BhZ2F0aW9uKCkpfX0sdC5wcm90b3R5cGUuX29uTW91c2VMZWF2ZT1mdW5jdGlvbihlKXt0aGlzLl9jdXJyZW50Wm9uZSYmKHRoaXMuX2N1cnJlbnRab25lLmxlYXZlQ2FsbGJhY2soKSx0aGlzLl9jdXJyZW50Wm9uZT12b2lkIDAsdGhpcy5fdG9vbHRpcFRpbWVvdXQmJmNsZWFyVGltZW91dCh0aGlzLl90b29sdGlwVGltZW91dCkpfSx0LnByb3RvdHlwZS5fb25DbGljaz1mdW5jdGlvbihlKXt2YXIgdD10aGlzLl9maW5kWm9uZUV2ZW50QXQoZSkscj10aGlzLl9nZXRTZWxlY3Rpb25MZW5ndGgoKTt0JiZyPT09dGhpcy5faW5pdGlhbFNlbGVjdGlvbkxlbmd0aCYmKHQuY2xpY2tDYWxsYmFjayhlKSxlLnByZXZlbnREZWZhdWx0KCksZS5zdG9wSW1tZWRpYXRlUHJvcGFnYXRpb24oKSl9LHQucHJvdG90eXBlLl9nZXRTZWxlY3Rpb25MZW5ndGg9ZnVuY3Rpb24oKXt2YXIgZT10aGlzLl9zZWxlY3Rpb25TZXJ2aWNlLnNlbGVjdGlvblRleHQ7cmV0dXJuIGU/ZS5sZW5ndGg6MH0sdC5wcm90b3R5cGUuX2ZpbmRab25lRXZlbnRBdD1mdW5jdGlvbihlKXt2YXIgdD10aGlzLl9tb3VzZVNlcnZpY2UuZ2V0Q29vcmRzKGUsdGhpcy5fc2NyZWVuRWxlbWVudCx0aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHMsdGhpcy5fYnVmZmVyU2VydmljZS5yb3dzKTtpZih0KWZvcih2YXIgcj10WzBdLGk9dFsxXSxuPTA7bjx0aGlzLl96b25lcy5sZW5ndGg7bisrKXt2YXIgbz10aGlzLl96b25lc1tuXTtpZihvLnkxPT09by55Mil7aWYoaT09PW8ueTEmJnI+PW8ueDEmJnI8by54MilyZXR1cm4gb31lbHNlIGlmKGk9PT1vLnkxJiZyPj1vLngxfHxpPT09by55MiYmcjxvLngyfHxpPm8ueTEmJmk8by55MilyZXR1cm4gb319LG8oW3MoMix1LklCdWZmZXJTZXJ2aWNlKSxzKDMsbC5JTW91c2VTZXJ2aWNlKSxzKDQsbC5JU2VsZWN0aW9uU2VydmljZSkscyg1LHUuSU9wdGlvbnNTZXJ2aWNlKV0sdCl9KGEuRGlzcG9zYWJsZSk7dC5Nb3VzZVpvbmVNYW5hZ2VyPWh9LDYxOTM6KGUsdCk9PntPYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC5SZW5kZXJEZWJvdW5jZXI9dm9pZCAwO3ZhciByPWZ1bmN0aW9uKCl7ZnVuY3Rpb24gZShlKXt0aGlzLl9yZW5kZXJDYWxsYmFjaz1lfXJldHVybiBlLnByb3RvdHlwZS5kaXNwb3NlPWZ1bmN0aW9uKCl7dGhpcy5fYW5pbWF0aW9uRnJhbWUmJih3aW5kb3cuY2FuY2VsQW5pbWF0aW9uRnJhbWUodGhpcy5fYW5pbWF0aW9uRnJhbWUpLHRoaXMuX2FuaW1hdGlvbkZyYW1lPXZvaWQgMCl9LGUucHJvdG90eXBlLnJlZnJlc2g9ZnVuY3Rpb24oZSx0LHIpe3ZhciBpPXRoaXM7dGhpcy5fcm93Q291bnQ9cixlPXZvaWQgMCE9PWU/ZTowLHQ9dm9pZCAwIT09dD90OnRoaXMuX3Jvd0NvdW50LTEsdGhpcy5fcm93U3RhcnQ9dm9pZCAwIT09dGhpcy5fcm93U3RhcnQ/TWF0aC5taW4odGhpcy5fcm93U3RhcnQsZSk6ZSx0aGlzLl9yb3dFbmQ9dm9pZCAwIT09dGhpcy5fcm93RW5kP01hdGgubWF4KHRoaXMuX3Jvd0VuZCx0KTp0LHRoaXMuX2FuaW1hdGlvbkZyYW1lfHwodGhpcy5fYW5pbWF0aW9uRnJhbWU9d2luZG93LnJlcXVlc3RBbmltYXRpb25GcmFtZSgoZnVuY3Rpb24oKXtyZXR1cm4gaS5faW5uZXJSZWZyZXNoKCl9KSkpfSxlLnByb3RvdHlwZS5faW5uZXJSZWZyZXNoPWZ1bmN0aW9uKCl7aWYodm9pZCAwIT09dGhpcy5fcm93U3RhcnQmJnZvaWQgMCE9PXRoaXMuX3Jvd0VuZCYmdm9pZCAwIT09dGhpcy5fcm93Q291bnQpe3ZhciBlPU1hdGgubWF4KHRoaXMuX3Jvd1N0YXJ0LDApLHQ9TWF0aC5taW4odGhpcy5fcm93RW5kLHRoaXMuX3Jvd0NvdW50LTEpO3RoaXMuX3Jvd1N0YXJ0PXZvaWQgMCx0aGlzLl9yb3dFbmQ9dm9pZCAwLHRoaXMuX2FuaW1hdGlvbkZyYW1lPXZvaWQgMCx0aGlzLl9yZW5kZXJDYWxsYmFjayhlLHQpfX0sZX0oKTt0LlJlbmRlckRlYm91bmNlcj1yfSw1NTk2OmZ1bmN0aW9uKGUsdCxyKXt2YXIgaSxuPXRoaXMmJnRoaXMuX19leHRlbmRzfHwoaT1mdW5jdGlvbihlLHQpe3JldHVybiBpPU9iamVjdC5zZXRQcm90b3R5cGVPZnx8e19fcHJvdG9fXzpbXX1pbnN0YW5jZW9mIEFycmF5JiZmdW5jdGlvbihlLHQpe2UuX19wcm90b19fPXR9fHxmdW5jdGlvbihlLHQpe2Zvcih2YXIgciBpbiB0KU9iamVjdC5wcm90b3R5cGUuaGFzT3duUHJvcGVydHkuY2FsbCh0LHIpJiYoZVtyXT10W3JdKX0saShlLHQpfSxmdW5jdGlvbihlLHQpe2lmKCJmdW5jdGlvbiIhPXR5cGVvZiB0JiZudWxsIT09dCl0aHJvdyBuZXcgVHlwZUVycm9yKCJDbGFzcyBleHRlbmRzIHZhbHVlICIrU3RyaW5nKHQpKyIgaXMgbm90IGEgY29uc3RydWN0b3Igb3IgbnVsbCIpO2Z1bmN0aW9uIHIoKXt0aGlzLmNvbnN0cnVjdG9yPWV9aShlLHQpLGUucHJvdG90eXBlPW51bGw9PT10P09iamVjdC5jcmVhdGUodCk6KHIucHJvdG90eXBlPXQucHJvdG90eXBlLG5ldyByKX0pO09iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LCJfX2VzTW9kdWxlIix7dmFsdWU6ITB9KSx0LlNjcmVlbkRwck1vbml0b3I9dm9pZCAwO3ZhciBvPWZ1bmN0aW9uKGUpe2Z1bmN0aW9uIHQoKXt2YXIgdD1udWxsIT09ZSYmZS5hcHBseSh0aGlzLGFyZ3VtZW50cyl8fHRoaXM7cmV0dXJuIHQuX2N1cnJlbnREZXZpY2VQaXhlbFJhdGlvPXdpbmRvdy5kZXZpY2VQaXhlbFJhdGlvLHR9cmV0dXJuIG4odCxlKSx0LnByb3RvdHlwZS5zZXRMaXN0ZW5lcj1mdW5jdGlvbihlKXt2YXIgdD10aGlzO3RoaXMuX2xpc3RlbmVyJiZ0aGlzLmNsZWFyTGlzdGVuZXIoKSx0aGlzLl9saXN0ZW5lcj1lLHRoaXMuX291dGVyTGlzdGVuZXI9ZnVuY3Rpb24oKXt0Ll9saXN0ZW5lciYmKHQuX2xpc3RlbmVyKHdpbmRvdy5kZXZpY2VQaXhlbFJhdGlvLHQuX2N1cnJlbnREZXZpY2VQaXhlbFJhdGlvKSx0Ll91cGRhdGVEcHIoKSl9LHRoaXMuX3VwZGF0ZURwcigpfSx0LnByb3RvdHlwZS5kaXNwb3NlPWZ1bmN0aW9uKCl7ZS5wcm90b3R5cGUuZGlzcG9zZS5jYWxsKHRoaXMpLHRoaXMuY2xlYXJMaXN0ZW5lcigpfSx0LnByb3RvdHlwZS5fdXBkYXRlRHByPWZ1bmN0aW9uKCl7dmFyIGU7dGhpcy5fb3V0ZXJMaXN0ZW5lciYmKG51bGw9PT0oZT10aGlzLl9yZXNvbHV0aW9uTWVkaWFNYXRjaExpc3QpfHx2b2lkIDA9PT1lfHxlLnJlbW92ZUxpc3RlbmVyKHRoaXMuX291dGVyTGlzdGVuZXIpLHRoaXMuX2N1cnJlbnREZXZpY2VQaXhlbFJhdGlvPXdpbmRvdy5kZXZpY2VQaXhlbFJhdGlvLHRoaXMuX3Jlc29sdXRpb25NZWRpYU1hdGNoTGlzdD13aW5kb3cubWF0Y2hNZWRpYSgic2NyZWVuIGFuZCAocmVzb2x1dGlvbjogIit3aW5kb3cuZGV2aWNlUGl4ZWxSYXRpbysiZHBweCkiKSx0aGlzLl9yZXNvbHV0aW9uTWVkaWFNYXRjaExpc3QuYWRkTGlzdGVuZXIodGhpcy5fb3V0ZXJMaXN0ZW5lcikpfSx0LnByb3RvdHlwZS5jbGVhckxpc3RlbmVyPWZ1bmN0aW9uKCl7dGhpcy5fcmVzb2x1dGlvbk1lZGlhTWF0Y2hMaXN0JiZ0aGlzLl9saXN0ZW5lciYmdGhpcy5fb3V0ZXJMaXN0ZW5lciYmKHRoaXMuX3Jlc29sdXRpb25NZWRpYU1hdGNoTGlzdC5yZW1vdmVMaXN0ZW5lcih0aGlzLl9vdXRlckxpc3RlbmVyKSx0aGlzLl9yZXNvbHV0aW9uTWVkaWFNYXRjaExpc3Q9dm9pZCAwLHRoaXMuX2xpc3RlbmVyPXZvaWQgMCx0aGlzLl9vdXRlckxpc3RlbmVyPXZvaWQgMCl9LHR9KHIoODQ0KS5EaXNwb3NhYmxlKTt0LlNjcmVlbkRwck1vbml0b3I9b30sMzIzNjpmdW5jdGlvbihlLHQscil7dmFyIGksbj10aGlzJiZ0aGlzLl9fZXh0ZW5kc3x8KGk9ZnVuY3Rpb24oZSx0KXtyZXR1cm4gaT1PYmplY3Quc2V0UHJvdG90eXBlT2Z8fHtfX3Byb3RvX186W119aW5zdGFuY2VvZiBBcnJheSYmZnVuY3Rpb24oZSx0KXtlLl9fcHJvdG9fXz10fXx8ZnVuY3Rpb24oZSx0KXtmb3IodmFyIHIgaW4gdClPYmplY3QucHJvdG90eXBlLmhhc093blByb3BlcnR5LmNhbGwodCxyKSYmKGVbcl09dFtyXSl9LGkoZSx0KX0sZnVuY3Rpb24oZSx0KXtpZigiZnVuY3Rpb24iIT10eXBlb2YgdCYmbnVsbCE9PXQpdGhyb3cgbmV3IFR5cGVFcnJvcigiQ2xhc3MgZXh0ZW5kcyB2YWx1ZSAiK1N0cmluZyh0KSsiIGlzIG5vdCBhIGNvbnN0cnVjdG9yIG9yIG51bGwiKTtmdW5jdGlvbiByKCl7dGhpcy5jb25zdHJ1Y3Rvcj1lfWkoZSx0KSxlLnByb3RvdHlwZT1udWxsPT09dD9PYmplY3QuY3JlYXRlKHQpOihyLnByb3RvdHlwZT10LnByb3RvdHlwZSxuZXcgcil9KTtPYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC5UZXJtaW5hbD12b2lkIDA7dmFyIG89cigyOTUwKSxzPXIoMTY4MCksYT1yKDM2MTQpLGM9cigyNTg0KSxsPXIoNTQzNSksdT1yKDM1MjUpLGg9cigzNTUxKSxmPXIoOTMxMiksXz1yKDYxMTQpLGQ9cigzNjU2KSxwPXIoOTA0Miksdj1yKDM1NyksZz1yKDY5NTQpLHk9cig0NTY3KSxtPXIoMTI5NiksYj1yKDczOTkpLFM9cig4NDYwKSxDPXIoODQzNyksdz1yKDU2ODApLEw9cigzMjMwKSxFPXIoNDcyNSkseD1yKDQyOCksQT1yKDg5MzQpLGs9cig2NDY1KSxNPXIoNTExNCksUj1yKDg5NjkpLFQ9cig0Nzc0KSxPPXIoNDI2OSksQj1yKDU5NDEpLEQ9InVuZGVmaW5lZCIhPXR5cGVvZiB3aW5kb3c/d2luZG93LmRvY3VtZW50Om51bGwsUD1mdW5jdGlvbihlKXtmdW5jdGlvbiB0KHQpe3ZvaWQgMD09PXQmJih0PXt9KTt2YXIgcj1lLmNhbGwodGhpcyx0KXx8dGhpcztyZXR1cm4gci5icm93c2VyPV8sci5fa2V5RG93bkhhbmRsZWQ9ITEsci5fa2V5UHJlc3NIYW5kbGVkPSExLHIuX3VucHJvY2Vzc2VkRGVhZEtleT0hMSxyLl9vbkN1cnNvck1vdmU9bmV3IFMuRXZlbnRFbWl0dGVyLHIuX29uS2V5PW5ldyBTLkV2ZW50RW1pdHRlcixyLl9vblJlbmRlcj1uZXcgUy5FdmVudEVtaXR0ZXIsci5fb25TZWxlY3Rpb25DaGFuZ2U9bmV3IFMuRXZlbnRFbWl0dGVyLHIuX29uVGl0bGVDaGFuZ2U9bmV3IFMuRXZlbnRFbWl0dGVyLHIuX29uQmVsbD1uZXcgUy5FdmVudEVtaXR0ZXIsci5fb25Gb2N1cz1uZXcgUy5FdmVudEVtaXR0ZXIsci5fb25CbHVyPW5ldyBTLkV2ZW50RW1pdHRlcixyLl9vbkExMXlDaGFyRW1pdHRlcj1uZXcgUy5FdmVudEVtaXR0ZXIsci5fb25BMTF5VGFiRW1pdHRlcj1uZXcgUy5FdmVudEVtaXR0ZXIsci5fc2V0dXAoKSxyLmxpbmtpZmllcj1yLl9pbnN0YW50aWF0aW9uU2VydmljZS5jcmVhdGVJbnN0YW5jZShoLkxpbmtpZmllciksci5saW5raWZpZXIyPXIucmVnaXN0ZXIoci5faW5zdGFudGlhdGlvblNlcnZpY2UuY3JlYXRlSW5zdGFuY2Uoay5MaW5raWZpZXIyKSksci5yZWdpc3RlcihyLl9pbnB1dEhhbmRsZXIub25SZXF1ZXN0QmVsbCgoZnVuY3Rpb24oKXtyZXR1cm4gci5iZWxsKCl9KSkpLHIucmVnaXN0ZXIoci5faW5wdXRIYW5kbGVyLm9uUmVxdWVzdFJlZnJlc2hSb3dzKChmdW5jdGlvbihlLHQpe3JldHVybiByLnJlZnJlc2goZSx0KX0pKSksci5yZWdpc3RlcihyLl9pbnB1dEhhbmRsZXIub25SZXF1ZXN0U2VuZEZvY3VzKChmdW5jdGlvbigpe3JldHVybiByLl9yZXBvcnRGb2N1cygpfSkpKSxyLnJlZ2lzdGVyKHIuX2lucHV0SGFuZGxlci5vblJlcXVlc3RSZXNldCgoZnVuY3Rpb24oKXtyZXR1cm4gci5yZXNldCgpfSkpKSxyLnJlZ2lzdGVyKHIuX2lucHV0SGFuZGxlci5vblJlcXVlc3RXaW5kb3dzT3B0aW9uc1JlcG9ydCgoZnVuY3Rpb24oZSl7cmV0dXJuIHIuX3JlcG9ydFdpbmRvd3NPcHRpb25zKGUpfSkpKSxyLnJlZ2lzdGVyKHIuX2lucHV0SGFuZGxlci5vbkNvbG9yKChmdW5jdGlvbihlKXtyZXR1cm4gci5faGFuZGxlQ29sb3JFdmVudChlKX0pKSksci5yZWdpc3RlcigoMCxTLmZvcndhcmRFdmVudCkoci5faW5wdXRIYW5kbGVyLm9uQ3Vyc29yTW92ZSxyLl9vbkN1cnNvck1vdmUpKSxyLnJlZ2lzdGVyKCgwLFMuZm9yd2FyZEV2ZW50KShyLl9pbnB1dEhhbmRsZXIub25UaXRsZUNoYW5nZSxyLl9vblRpdGxlQ2hhbmdlKSksci5yZWdpc3RlcigoMCxTLmZvcndhcmRFdmVudCkoci5faW5wdXRIYW5kbGVyLm9uQTExeUNoYXIsci5fb25BMTF5Q2hhckVtaXR0ZXIpKSxyLnJlZ2lzdGVyKCgwLFMuZm9yd2FyZEV2ZW50KShyLl9pbnB1dEhhbmRsZXIub25BMTF5VGFiLHIuX29uQTExeVRhYkVtaXR0ZXIpKSxyLnJlZ2lzdGVyKHIuX2J1ZmZlclNlcnZpY2Uub25SZXNpemUoKGZ1bmN0aW9uKGUpe3JldHVybiByLl9hZnRlclJlc2l6ZShlLmNvbHMsZS5yb3dzKX0pKSkscn1yZXR1cm4gbih0LGUpLE9iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LnByb3RvdHlwZSwib25DdXJzb3JNb3ZlIix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX29uQ3Vyc29yTW92ZS5ldmVudH0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxPYmplY3QuZGVmaW5lUHJvcGVydHkodC5wcm90b3R5cGUsIm9uS2V5Iix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX29uS2V5LmV2ZW50fSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLE9iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LnByb3RvdHlwZSwib25SZW5kZXIiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fb25SZW5kZXIuZXZlbnR9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KHQucHJvdG90eXBlLCJvblNlbGVjdGlvbkNoYW5nZSIse2dldDpmdW5jdGlvbigpe3JldHVybiB0aGlzLl9vblNlbGVjdGlvbkNoYW5nZS5ldmVudH0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxPYmplY3QuZGVmaW5lUHJvcGVydHkodC5wcm90b3R5cGUsIm9uVGl0bGVDaGFuZ2UiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fb25UaXRsZUNoYW5nZS5ldmVudH0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxPYmplY3QuZGVmaW5lUHJvcGVydHkodC5wcm90b3R5cGUsIm9uQmVsbCIse2dldDpmdW5jdGlvbigpe3JldHVybiB0aGlzLl9vbkJlbGwuZXZlbnR9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KHQucHJvdG90eXBlLCJvbkZvY3VzIix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX29uRm9jdXMuZXZlbnR9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KHQucHJvdG90eXBlLCJvbkJsdXIiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fb25CbHVyLmV2ZW50fSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLE9iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LnByb3RvdHlwZSwib25BMTF5Q2hhciIse2dldDpmdW5jdGlvbigpe3JldHVybiB0aGlzLl9vbkExMXlDaGFyRW1pdHRlci5ldmVudH0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxPYmplY3QuZGVmaW5lUHJvcGVydHkodC5wcm90b3R5cGUsIm9uQTExeVRhYiIse2dldDpmdW5jdGlvbigpe3JldHVybiB0aGlzLl9vbkExMXlUYWJFbWl0dGVyLmV2ZW50fSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLHQucHJvdG90eXBlLl9oYW5kbGVDb2xvckV2ZW50PWZ1bmN0aW9uKGUpe3ZhciB0LHI7aWYodGhpcy5fY29sb3JNYW5hZ2VyKXtmb3IodmFyIGk9MCxuPWU7aTxuLmxlbmd0aDtpKyspe3ZhciBvPW5baV0scz12b2lkIDAsYT0iIjtzd2l0Y2goby5pbmRleCl7Y2FzZSAyNTY6cz0iZm9yZWdyb3VuZCIsYT0iMTAiO2JyZWFrO2Nhc2UgMjU3OnM9ImJhY2tncm91bmQiLGE9IjExIjticmVhaztjYXNlIDI1ODpzPSJjdXJzb3IiLGE9IjEyIjticmVhaztkZWZhdWx0OnM9ImFuc2kiLGE9IjQ7IitvLmluZGV4fWlmKHMpc3dpdGNoKG8udHlwZSl7Y2FzZSAwOnZhciBsPVQuY29sb3IudG9Db2xvclJHQigiYW5zaSI9PT1zP3RoaXMuX2NvbG9yTWFuYWdlci5jb2xvcnMuYW5zaVtvLmluZGV4XTp0aGlzLl9jb2xvck1hbmFnZXIuY29sb3JzW3NdKTt0aGlzLmNvcmVTZXJ2aWNlLnRyaWdnZXJEYXRhRXZlbnQoYy5DMC5FU0MrIl0iK2ErIjsiKygwLEIudG9SZ2JTdHJpbmcpKGwpK2MuQzAuQkVMKTticmVhaztjYXNlIDE6ImFuc2kiPT09cz90aGlzLl9jb2xvck1hbmFnZXIuY29sb3JzLmFuc2lbby5pbmRleF09VC5yZ2JhLnRvQ29sb3IuYXBwbHkoVC5yZ2JhLG8uY29sb3IpOnRoaXMuX2NvbG9yTWFuYWdlci5jb2xvcnNbc109VC5yZ2JhLnRvQ29sb3IuYXBwbHkoVC5yZ2JhLG8uY29sb3IpO2JyZWFrO2Nhc2UgMjp0aGlzLl9jb2xvck1hbmFnZXIucmVzdG9yZUNvbG9yKG8uaW5kZXgpfX1udWxsPT09KHQ9dGhpcy5fcmVuZGVyU2VydmljZSl8fHZvaWQgMD09PXR8fHQuc2V0Q29sb3JzKHRoaXMuX2NvbG9yTWFuYWdlci5jb2xvcnMpLG51bGw9PT0ocj10aGlzLnZpZXdwb3J0KXx8dm9pZCAwPT09cnx8ci5vblRoZW1lQ2hhbmdlKHRoaXMuX2NvbG9yTWFuYWdlci5jb2xvcnMpfX0sdC5wcm90b3R5cGUuZGlzcG9zZT1mdW5jdGlvbigpe3ZhciB0LHIsaTt0aGlzLl9pc0Rpc3Bvc2VkfHwoZS5wcm90b3R5cGUuZGlzcG9zZS5jYWxsKHRoaXMpLG51bGw9PT0odD10aGlzLl9yZW5kZXJTZXJ2aWNlKXx8dm9pZCAwPT09dHx8dC5kaXNwb3NlKCksdGhpcy5fY3VzdG9tS2V5RXZlbnRIYW5kbGVyPXZvaWQgMCx0aGlzLndyaXRlPWZ1bmN0aW9uKCl7fSxudWxsPT09KGk9bnVsbD09PShyPXRoaXMuZWxlbWVudCl8fHZvaWQgMD09PXI/dm9pZCAwOnIucGFyZW50Tm9kZSl8fHZvaWQgMD09PWl8fGkucmVtb3ZlQ2hpbGQodGhpcy5lbGVtZW50KSl9LHQucHJvdG90eXBlLl9zZXR1cD1mdW5jdGlvbigpe2UucHJvdG90eXBlLl9zZXR1cC5jYWxsKHRoaXMpLHRoaXMuX2N1c3RvbUtleUV2ZW50SGFuZGxlcj12b2lkIDB9LE9iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LnByb3RvdHlwZSwiYnVmZmVyIix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuYnVmZmVycy5hY3RpdmV9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksdC5wcm90b3R5cGUuZm9jdXM9ZnVuY3Rpb24oKXt0aGlzLnRleHRhcmVhJiZ0aGlzLnRleHRhcmVhLmZvY3VzKHtwcmV2ZW50U2Nyb2xsOiEwfSl9LHQucHJvdG90eXBlLl91cGRhdGVPcHRpb25zPWZ1bmN0aW9uKHQpe3ZhciByLGksbixvO3N3aXRjaChlLnByb3RvdHlwZS5fdXBkYXRlT3B0aW9ucy5jYWxsKHRoaXMsdCksdCl7Y2FzZSJmb250RmFtaWx5IjpjYXNlImZvbnRTaXplIjpudWxsPT09KHI9dGhpcy5fcmVuZGVyU2VydmljZSl8fHZvaWQgMD09PXJ8fHIuY2xlYXIoKSxudWxsPT09KGk9dGhpcy5fY2hhclNpemVTZXJ2aWNlKXx8dm9pZCAwPT09aXx8aS5tZWFzdXJlKCk7YnJlYWs7Y2FzZSJjdXJzb3JCbGluayI6Y2FzZSJjdXJzb3JTdHlsZSI6dGhpcy5yZWZyZXNoKHRoaXMuYnVmZmVyLnksdGhpcy5idWZmZXIueSk7YnJlYWs7Y2FzZSJjdXN0b21HbHlwaHMiOmNhc2UiZHJhd0JvbGRUZXh0SW5CcmlnaHRDb2xvcnMiOmNhc2UibGV0dGVyU3BhY2luZyI6Y2FzZSJsaW5lSGVpZ2h0IjpjYXNlImZvbnRXZWlnaHQiOmNhc2UiZm9udFdlaWdodEJvbGQiOmNhc2UibWluaW11bUNvbnRyYXN0UmF0aW8iOnRoaXMuX3JlbmRlclNlcnZpY2UmJih0aGlzLl9yZW5kZXJTZXJ2aWNlLmNsZWFyKCksdGhpcy5fcmVuZGVyU2VydmljZS5vblJlc2l6ZSh0aGlzLmNvbHMsdGhpcy5yb3dzKSx0aGlzLnJlZnJlc2goMCx0aGlzLnJvd3MtMSkpO2JyZWFrO2Nhc2UicmVuZGVyZXJUeXBlIjp0aGlzLl9yZW5kZXJTZXJ2aWNlJiYodGhpcy5fcmVuZGVyU2VydmljZS5zZXRSZW5kZXJlcih0aGlzLl9jcmVhdGVSZW5kZXJlcigpKSx0aGlzLl9yZW5kZXJTZXJ2aWNlLm9uUmVzaXplKHRoaXMuY29scyx0aGlzLnJvd3MpKTticmVhaztjYXNlInNjcm9sbGJhY2siOm51bGw9PT0obj10aGlzLnZpZXdwb3J0KXx8dm9pZCAwPT09bnx8bi5zeW5jU2Nyb2xsQXJlYSgpO2JyZWFrO2Nhc2Uic2NyZWVuUmVhZGVyTW9kZSI6dGhpcy5vcHRpb25zU2VydmljZS5vcHRpb25zLnNjcmVlblJlYWRlck1vZGU/IXRoaXMuX2FjY2Vzc2liaWxpdHlNYW5hZ2VyJiZ0aGlzLl9yZW5kZXJTZXJ2aWNlJiYodGhpcy5fYWNjZXNzaWJpbGl0eU1hbmFnZXI9bmV3IHkuQWNjZXNzaWJpbGl0eU1hbmFnZXIodGhpcyx0aGlzLl9yZW5kZXJTZXJ2aWNlKSk6KG51bGw9PT0obz10aGlzLl9hY2Nlc3NpYmlsaXR5TWFuYWdlcil8fHZvaWQgMD09PW98fG8uZGlzcG9zZSgpLHRoaXMuX2FjY2Vzc2liaWxpdHlNYW5hZ2VyPXZvaWQgMCk7YnJlYWs7Y2FzZSJ0YWJTdG9wV2lkdGgiOnRoaXMuYnVmZmVycy5zZXR1cFRhYlN0b3BzKCk7YnJlYWs7Y2FzZSJ0aGVtZSI6dGhpcy5fc2V0VGhlbWUodGhpcy5vcHRpb25zU2VydmljZS5vcHRpb25zLnRoZW1lKX19LHQucHJvdG90eXBlLl9vblRleHRBcmVhRm9jdXM9ZnVuY3Rpb24oZSl7dGhpcy5jb3JlU2VydmljZS5kZWNQcml2YXRlTW9kZXMuc2VuZEZvY3VzJiZ0aGlzLmNvcmVTZXJ2aWNlLnRyaWdnZXJEYXRhRXZlbnQoYy5DMC5FU0MrIltJIiksdGhpcy51cGRhdGVDdXJzb3JTdHlsZShlKSx0aGlzLmVsZW1lbnQuY2xhc3NMaXN0LmFkZCgiZm9jdXMiKSx0aGlzLl9zaG93Q3Vyc29yKCksdGhpcy5fb25Gb2N1cy5maXJlKCl9LHQucHJvdG90eXBlLmJsdXI9ZnVuY3Rpb24oKXt2YXIgZTtyZXR1cm4gbnVsbD09PShlPXRoaXMudGV4dGFyZWEpfHx2b2lkIDA9PT1lP3ZvaWQgMDplLmJsdXIoKX0sdC5wcm90b3R5cGUuX29uVGV4dEFyZWFCbHVyPWZ1bmN0aW9uKCl7dGhpcy50ZXh0YXJlYS52YWx1ZT0iIix0aGlzLnJlZnJlc2godGhpcy5idWZmZXIueSx0aGlzLmJ1ZmZlci55KSx0aGlzLmNvcmVTZXJ2aWNlLmRlY1ByaXZhdGVNb2Rlcy5zZW5kRm9jdXMmJnRoaXMuY29yZVNlcnZpY2UudHJpZ2dlckRhdGFFdmVudChjLkMwLkVTQysiW08iKSx0aGlzLmVsZW1lbnQuY2xhc3NMaXN0LnJlbW92ZSgiZm9jdXMiKSx0aGlzLl9vbkJsdXIuZmlyZSgpfSx0LnByb3RvdHlwZS5fc3luY1RleHRBcmVhPWZ1bmN0aW9uKCl7aWYodGhpcy50ZXh0YXJlYSYmdGhpcy5idWZmZXIuaXNDdXJzb3JJblZpZXdwb3J0JiYhdGhpcy5fY29tcG9zaXRpb25IZWxwZXIuaXNDb21wb3NpbmcmJnRoaXMuX3JlbmRlclNlcnZpY2Upe3ZhciBlPXRoaXMuYnVmZmVyLnliYXNlK3RoaXMuYnVmZmVyLnksdD10aGlzLmJ1ZmZlci5saW5lcy5nZXQoZSk7aWYodCl7dmFyIHI9TWF0aC5taW4odGhpcy5idWZmZXIueCx0aGlzLmNvbHMtMSksaT10aGlzLl9yZW5kZXJTZXJ2aWNlLmRpbWVuc2lvbnMuYWN0dWFsQ2VsbEhlaWdodCxuPXQuZ2V0V2lkdGgociksbz10aGlzLl9yZW5kZXJTZXJ2aWNlLmRpbWVuc2lvbnMuYWN0dWFsQ2VsbFdpZHRoKm4scz10aGlzLmJ1ZmZlci55KnRoaXMuX3JlbmRlclNlcnZpY2UuZGltZW5zaW9ucy5hY3R1YWxDZWxsSGVpZ2h0LGE9cip0aGlzLl9yZW5kZXJTZXJ2aWNlLmRpbWVuc2lvbnMuYWN0dWFsQ2VsbFdpZHRoO3RoaXMudGV4dGFyZWEuc3R5bGUubGVmdD1hKyJweCIsdGhpcy50ZXh0YXJlYS5zdHlsZS50b3A9cysicHgiLHRoaXMudGV4dGFyZWEuc3R5bGUud2lkdGg9bysicHgiLHRoaXMudGV4dGFyZWEuc3R5bGUuaGVpZ2h0PWkrInB4Iix0aGlzLnRleHRhcmVhLnN0eWxlLmxpbmVIZWlnaHQ9aSsicHgiLHRoaXMudGV4dGFyZWEuc3R5bGUuekluZGV4PSItNSJ9fX0sdC5wcm90b3R5cGUuX2luaXRHbG9iYWw9ZnVuY3Rpb24oKXt2YXIgZT10aGlzO3RoaXMuX2JpbmRLZXlzKCksdGhpcy5yZWdpc3RlcigoMCxkLmFkZERpc3Bvc2FibGVEb21MaXN0ZW5lcikodGhpcy5lbGVtZW50LCJjb3B5IiwoZnVuY3Rpb24odCl7ZS5oYXNTZWxlY3Rpb24oKSYmKDAsYS5jb3B5SGFuZGxlcikodCxlLl9zZWxlY3Rpb25TZXJ2aWNlKX0pKSk7dmFyIHQ9ZnVuY3Rpb24odCl7cmV0dXJuKDAsYS5oYW5kbGVQYXN0ZUV2ZW50KSh0LGUudGV4dGFyZWEsZS5jb3JlU2VydmljZSl9O3RoaXMucmVnaXN0ZXIoKDAsZC5hZGREaXNwb3NhYmxlRG9tTGlzdGVuZXIpKHRoaXMudGV4dGFyZWEsInBhc3RlIix0KSksdGhpcy5yZWdpc3RlcigoMCxkLmFkZERpc3Bvc2FibGVEb21MaXN0ZW5lcikodGhpcy5lbGVtZW50LCJwYXN0ZSIsdCkpLF8uaXNGaXJlZm94P3RoaXMucmVnaXN0ZXIoKDAsZC5hZGREaXNwb3NhYmxlRG9tTGlzdGVuZXIpKHRoaXMuZWxlbWVudCwibW91c2Vkb3duIiwoZnVuY3Rpb24odCl7Mj09PXQuYnV0dG9uJiYoMCxhLnJpZ2h0Q2xpY2tIYW5kbGVyKSh0LGUudGV4dGFyZWEsZS5zY3JlZW5FbGVtZW50LGUuX3NlbGVjdGlvblNlcnZpY2UsZS5vcHRpb25zLnJpZ2h0Q2xpY2tTZWxlY3RzV29yZCl9KSkpOnRoaXMucmVnaXN0ZXIoKDAsZC5hZGREaXNwb3NhYmxlRG9tTGlzdGVuZXIpKHRoaXMuZWxlbWVudCwiY29udGV4dG1lbnUiLChmdW5jdGlvbih0KXsoMCxhLnJpZ2h0Q2xpY2tIYW5kbGVyKSh0LGUudGV4dGFyZWEsZS5zY3JlZW5FbGVtZW50LGUuX3NlbGVjdGlvblNlcnZpY2UsZS5vcHRpb25zLnJpZ2h0Q2xpY2tTZWxlY3RzV29yZCl9KSkpLF8uaXNMaW51eCYmdGhpcy5yZWdpc3RlcigoMCxkLmFkZERpc3Bvc2FibGVEb21MaXN0ZW5lcikodGhpcy5lbGVtZW50LCJhdXhjbGljayIsKGZ1bmN0aW9uKHQpezE9PT10LmJ1dHRvbiYmKDAsYS5tb3ZlVGV4dEFyZWFVbmRlck1vdXNlQ3Vyc29yKSh0LGUudGV4dGFyZWEsZS5zY3JlZW5FbGVtZW50KX0pKSl9LHQucHJvdG90eXBlLl9iaW5kS2V5cz1mdW5jdGlvbigpe3ZhciBlPXRoaXM7dGhpcy5yZWdpc3RlcigoMCxkLmFkZERpc3Bvc2FibGVEb21MaXN0ZW5lcikodGhpcy50ZXh0YXJlYSwia2V5dXAiLChmdW5jdGlvbih0KXtyZXR1cm4gZS5fa2V5VXAodCl9KSwhMCkpLHRoaXMucmVnaXN0ZXIoKDAsZC5hZGREaXNwb3NhYmxlRG9tTGlzdGVuZXIpKHRoaXMudGV4dGFyZWEsImtleWRvd24iLChmdW5jdGlvbih0KXtyZXR1cm4gZS5fa2V5RG93bih0KX0pLCEwKSksdGhpcy5yZWdpc3RlcigoMCxkLmFkZERpc3Bvc2FibGVEb21MaXN0ZW5lcikodGhpcy50ZXh0YXJlYSwia2V5cHJlc3MiLChmdW5jdGlvbih0KXtyZXR1cm4gZS5fa2V5UHJlc3ModCl9KSwhMCkpLHRoaXMucmVnaXN0ZXIoKDAsZC5hZGREaXNwb3NhYmxlRG9tTGlzdGVuZXIpKHRoaXMudGV4dGFyZWEsImNvbXBvc2l0aW9uc3RhcnQiLChmdW5jdGlvbigpe3JldHVybiBlLl9jb21wb3NpdGlvbkhlbHBlci5jb21wb3NpdGlvbnN0YXJ0KCl9KSkpLHRoaXMucmVnaXN0ZXIoKDAsZC5hZGREaXNwb3NhYmxlRG9tTGlzdGVuZXIpKHRoaXMudGV4dGFyZWEsImNvbXBvc2l0aW9udXBkYXRlIiwoZnVuY3Rpb24odCl7cmV0dXJuIGUuX2NvbXBvc2l0aW9uSGVscGVyLmNvbXBvc2l0aW9udXBkYXRlKHQpfSkpKSx0aGlzLnJlZ2lzdGVyKCgwLGQuYWRkRGlzcG9zYWJsZURvbUxpc3RlbmVyKSh0aGlzLnRleHRhcmVhLCJjb21wb3NpdGlvbmVuZCIsKGZ1bmN0aW9uKCl7cmV0dXJuIGUuX2NvbXBvc2l0aW9uSGVscGVyLmNvbXBvc2l0aW9uZW5kKCl9KSkpLHRoaXMucmVnaXN0ZXIoKDAsZC5hZGREaXNwb3NhYmxlRG9tTGlzdGVuZXIpKHRoaXMudGV4dGFyZWEsImlucHV0IiwoZnVuY3Rpb24odCl7cmV0dXJuIGUuX2lucHV0RXZlbnQodCl9KSwhMCkpLHRoaXMucmVnaXN0ZXIodGhpcy5vblJlbmRlcigoZnVuY3Rpb24oKXtyZXR1cm4gZS5fY29tcG9zaXRpb25IZWxwZXIudXBkYXRlQ29tcG9zaXRpb25FbGVtZW50cygpfSkpKSx0aGlzLnJlZ2lzdGVyKHRoaXMub25SZW5kZXIoKGZ1bmN0aW9uKHQpe3JldHVybiBlLl9xdWV1ZUxpbmtpZmljYXRpb24odC5zdGFydCx0LmVuZCl9KSkpfSx0LnByb3RvdHlwZS5vcGVuPWZ1bmN0aW9uKGUpe3ZhciB0PXRoaXM7aWYoIWUpdGhyb3cgbmV3IEVycm9yKCJUZXJtaW5hbCByZXF1aXJlcyBhIHBhcmVudCBlbGVtZW50LiIpO2UuaXNDb25uZWN0ZWR8fHRoaXMuX2xvZ1NlcnZpY2UuZGVidWcoIlRlcm1pbmFsLm9wZW4gd2FzIGNhbGxlZCBvbiBhbiBlbGVtZW50IHRoYXQgd2FzIG5vdCBhdHRhY2hlZCB0byB0aGUgRE9NIiksdGhpcy5fZG9jdW1lbnQ9ZS5vd25lckRvY3VtZW50LHRoaXMuZWxlbWVudD10aGlzLl9kb2N1bWVudC5jcmVhdGVFbGVtZW50KCJkaXYiKSx0aGlzLmVsZW1lbnQuZGlyPSJsdHIiLHRoaXMuZWxlbWVudC5jbGFzc0xpc3QuYWRkKCJ0ZXJtaW5hbCIpLHRoaXMuZWxlbWVudC5jbGFzc0xpc3QuYWRkKCJ4dGVybSIpLHRoaXMuZWxlbWVudC5zZXRBdHRyaWJ1dGUoInRhYmluZGV4IiwiMCIpLGUuYXBwZW5kQ2hpbGQodGhpcy5lbGVtZW50KTt2YXIgcj1ELmNyZWF0ZURvY3VtZW50RnJhZ21lbnQoKTt0aGlzLl92aWV3cG9ydEVsZW1lbnQ9RC5jcmVhdGVFbGVtZW50KCJkaXYiKSx0aGlzLl92aWV3cG9ydEVsZW1lbnQuY2xhc3NMaXN0LmFkZCgieHRlcm0tdmlld3BvcnQiKSxyLmFwcGVuZENoaWxkKHRoaXMuX3ZpZXdwb3J0RWxlbWVudCksdGhpcy5fdmlld3BvcnRTY3JvbGxBcmVhPUQuY3JlYXRlRWxlbWVudCgiZGl2IiksdGhpcy5fdmlld3BvcnRTY3JvbGxBcmVhLmNsYXNzTGlzdC5hZGQoInh0ZXJtLXNjcm9sbC1hcmVhIiksdGhpcy5fdmlld3BvcnRFbGVtZW50LmFwcGVuZENoaWxkKHRoaXMuX3ZpZXdwb3J0U2Nyb2xsQXJlYSksdGhpcy5zY3JlZW5FbGVtZW50PUQuY3JlYXRlRWxlbWVudCgiZGl2IiksdGhpcy5zY3JlZW5FbGVtZW50LmNsYXNzTGlzdC5hZGQoInh0ZXJtLXNjcmVlbiIpLHRoaXMuX2hlbHBlckNvbnRhaW5lcj1ELmNyZWF0ZUVsZW1lbnQoImRpdiIpLHRoaXMuX2hlbHBlckNvbnRhaW5lci5jbGFzc0xpc3QuYWRkKCJ4dGVybS1oZWxwZXJzIiksdGhpcy5zY3JlZW5FbGVtZW50LmFwcGVuZENoaWxkKHRoaXMuX2hlbHBlckNvbnRhaW5lciksci5hcHBlbmRDaGlsZCh0aGlzLnNjcmVlbkVsZW1lbnQpLHRoaXMudGV4dGFyZWE9RC5jcmVhdGVFbGVtZW50KCJ0ZXh0YXJlYSIpLHRoaXMudGV4dGFyZWEuY2xhc3NMaXN0LmFkZCgieHRlcm0taGVscGVyLXRleHRhcmVhIiksdGhpcy50ZXh0YXJlYS5zZXRBdHRyaWJ1dGUoImFyaWEtbGFiZWwiLHAucHJvbXB0TGFiZWwpLHRoaXMudGV4dGFyZWEuc2V0QXR0cmlidXRlKCJhcmlhLW11bHRpbGluZSIsImZhbHNlIiksdGhpcy50ZXh0YXJlYS5zZXRBdHRyaWJ1dGUoImF1dG9jb3JyZWN0Iiwib2ZmIiksdGhpcy50ZXh0YXJlYS5zZXRBdHRyaWJ1dGUoImF1dG9jYXBpdGFsaXplIiwib2ZmIiksdGhpcy50ZXh0YXJlYS5zZXRBdHRyaWJ1dGUoInNwZWxsY2hlY2siLCJmYWxzZSIpLHRoaXMudGV4dGFyZWEudGFiSW5kZXg9MCx0aGlzLnJlZ2lzdGVyKCgwLGQuYWRkRGlzcG9zYWJsZURvbUxpc3RlbmVyKSh0aGlzLnRleHRhcmVhLCJmb2N1cyIsKGZ1bmN0aW9uKGUpe3JldHVybiB0Ll9vblRleHRBcmVhRm9jdXMoZSl9KSkpLHRoaXMucmVnaXN0ZXIoKDAsZC5hZGREaXNwb3NhYmxlRG9tTGlzdGVuZXIpKHRoaXMudGV4dGFyZWEsImJsdXIiLChmdW5jdGlvbigpe3JldHVybiB0Ll9vblRleHRBcmVhQmx1cigpfSkpKSx0aGlzLl9oZWxwZXJDb250YWluZXIuYXBwZW5kQ2hpbGQodGhpcy50ZXh0YXJlYSk7dmFyIGk9dGhpcy5faW5zdGFudGlhdGlvblNlcnZpY2UuY3JlYXRlSW5zdGFuY2UoTS5Db3JlQnJvd3NlclNlcnZpY2UsdGhpcy50ZXh0YXJlYSk7dGhpcy5faW5zdGFudGlhdGlvblNlcnZpY2Uuc2V0U2VydmljZShFLklDb3JlQnJvd3NlclNlcnZpY2UsaSksdGhpcy5fY2hhclNpemVTZXJ2aWNlPXRoaXMuX2luc3RhbnRpYXRpb25TZXJ2aWNlLmNyZWF0ZUluc3RhbmNlKHguQ2hhclNpemVTZXJ2aWNlLHRoaXMuX2RvY3VtZW50LHRoaXMuX2hlbHBlckNvbnRhaW5lciksdGhpcy5faW5zdGFudGlhdGlvblNlcnZpY2Uuc2V0U2VydmljZShFLklDaGFyU2l6ZVNlcnZpY2UsdGhpcy5fY2hhclNpemVTZXJ2aWNlKSx0aGlzLl90aGVtZT10aGlzLm9wdGlvbnMudGhlbWV8fHRoaXMuX3RoZW1lLHRoaXMuX2NvbG9yTWFuYWdlcj1uZXcgdy5Db2xvck1hbmFnZXIoRCx0aGlzLm9wdGlvbnMuYWxsb3dUcmFuc3BhcmVuY3kpLHRoaXMucmVnaXN0ZXIodGhpcy5vcHRpb25zU2VydmljZS5vbk9wdGlvbkNoYW5nZSgoZnVuY3Rpb24oZSl7cmV0dXJuIHQuX2NvbG9yTWFuYWdlci5vbk9wdGlvbnNDaGFuZ2UoZSl9KSkpLHRoaXMuX2NvbG9yTWFuYWdlci5zZXRUaGVtZSh0aGlzLl90aGVtZSksdGhpcy5fY2hhcmFjdGVySm9pbmVyU2VydmljZT10aGlzLl9pbnN0YW50aWF0aW9uU2VydmljZS5jcmVhdGVJbnN0YW5jZShPLkNoYXJhY3RlckpvaW5lclNlcnZpY2UpLHRoaXMuX2luc3RhbnRpYXRpb25TZXJ2aWNlLnNldFNlcnZpY2UoRS5JQ2hhcmFjdGVySm9pbmVyU2VydmljZSx0aGlzLl9jaGFyYWN0ZXJKb2luZXJTZXJ2aWNlKTt2YXIgbj10aGlzLl9jcmVhdGVSZW5kZXJlcigpO3RoaXMuX3JlbmRlclNlcnZpY2U9dGhpcy5yZWdpc3Rlcih0aGlzLl9pbnN0YW50aWF0aW9uU2VydmljZS5jcmVhdGVJbnN0YW5jZShMLlJlbmRlclNlcnZpY2Usbix0aGlzLnJvd3MsdGhpcy5zY3JlZW5FbGVtZW50KSksdGhpcy5faW5zdGFudGlhdGlvblNlcnZpY2Uuc2V0U2VydmljZShFLklSZW5kZXJTZXJ2aWNlLHRoaXMuX3JlbmRlclNlcnZpY2UpLHRoaXMucmVnaXN0ZXIodGhpcy5fcmVuZGVyU2VydmljZS5vblJlbmRlcmVkQnVmZmVyQ2hhbmdlKChmdW5jdGlvbihlKXtyZXR1cm4gdC5fb25SZW5kZXIuZmlyZShlKX0pKSksdGhpcy5vblJlc2l6ZSgoZnVuY3Rpb24oZSl7cmV0dXJuIHQuX3JlbmRlclNlcnZpY2UucmVzaXplKGUuY29scyxlLnJvd3MpfSkpLHRoaXMuX2NvbXBvc2l0aW9uVmlldz1ELmNyZWF0ZUVsZW1lbnQoImRpdiIpLHRoaXMuX2NvbXBvc2l0aW9uVmlldy5jbGFzc0xpc3QuYWRkKCJjb21wb3NpdGlvbi12aWV3IiksdGhpcy5fY29tcG9zaXRpb25IZWxwZXI9dGhpcy5faW5zdGFudGlhdGlvblNlcnZpY2UuY3JlYXRlSW5zdGFuY2Uoby5Db21wb3NpdGlvbkhlbHBlcix0aGlzLnRleHRhcmVhLHRoaXMuX2NvbXBvc2l0aW9uVmlldyksdGhpcy5faGVscGVyQ29udGFpbmVyLmFwcGVuZENoaWxkKHRoaXMuX2NvbXBvc2l0aW9uVmlldyksdGhpcy5lbGVtZW50LmFwcGVuZENoaWxkKHIpLHRoaXMuX3NvdW5kU2VydmljZT10aGlzLl9pbnN0YW50aWF0aW9uU2VydmljZS5jcmVhdGVJbnN0YW5jZSh2LlNvdW5kU2VydmljZSksdGhpcy5faW5zdGFudGlhdGlvblNlcnZpY2Uuc2V0U2VydmljZShFLklTb3VuZFNlcnZpY2UsdGhpcy5fc291bmRTZXJ2aWNlKSx0aGlzLl9tb3VzZVNlcnZpY2U9dGhpcy5faW5zdGFudGlhdGlvblNlcnZpY2UuY3JlYXRlSW5zdGFuY2UoQS5Nb3VzZVNlcnZpY2UpLHRoaXMuX2luc3RhbnRpYXRpb25TZXJ2aWNlLnNldFNlcnZpY2UoRS5JTW91c2VTZXJ2aWNlLHRoaXMuX21vdXNlU2VydmljZSksdGhpcy52aWV3cG9ydD10aGlzLl9pbnN0YW50aWF0aW9uU2VydmljZS5jcmVhdGVJbnN0YW5jZShzLlZpZXdwb3J0LChmdW5jdGlvbihlKXtyZXR1cm4gdC5zY3JvbGxMaW5lcyhlLCEwLDEpfSksdGhpcy5fdmlld3BvcnRFbGVtZW50LHRoaXMuX3ZpZXdwb3J0U2Nyb2xsQXJlYSx0aGlzLmVsZW1lbnQpLHRoaXMudmlld3BvcnQub25UaGVtZUNoYW5nZSh0aGlzLl9jb2xvck1hbmFnZXIuY29sb3JzKSx0aGlzLnJlZ2lzdGVyKHRoaXMuX2lucHV0SGFuZGxlci5vblJlcXVlc3RTeW5jU2Nyb2xsQmFyKChmdW5jdGlvbigpe3JldHVybiB0LnZpZXdwb3J0LnN5bmNTY3JvbGxBcmVhKCl9KSkpLHRoaXMucmVnaXN0ZXIodGhpcy52aWV3cG9ydCksdGhpcy5yZWdpc3Rlcih0aGlzLm9uQ3Vyc29yTW92ZSgoZnVuY3Rpb24oKXt0Ll9yZW5kZXJTZXJ2aWNlLm9uQ3Vyc29yTW92ZSgpLHQuX3N5bmNUZXh0QXJlYSgpfSkpKSx0aGlzLnJlZ2lzdGVyKHRoaXMub25SZXNpemUoKGZ1bmN0aW9uKCl7cmV0dXJuIHQuX3JlbmRlclNlcnZpY2Uub25SZXNpemUodC5jb2xzLHQucm93cyl9KSkpLHRoaXMucmVnaXN0ZXIodGhpcy5vbkJsdXIoKGZ1bmN0aW9uKCl7cmV0dXJuIHQuX3JlbmRlclNlcnZpY2Uub25CbHVyKCl9KSkpLHRoaXMucmVnaXN0ZXIodGhpcy5vbkZvY3VzKChmdW5jdGlvbigpe3JldHVybiB0Ll9yZW5kZXJTZXJ2aWNlLm9uRm9jdXMoKX0pKSksdGhpcy5yZWdpc3Rlcih0aGlzLl9yZW5kZXJTZXJ2aWNlLm9uRGltZW5zaW9uc0NoYW5nZSgoZnVuY3Rpb24oKXtyZXR1cm4gdC52aWV3cG9ydC5zeW5jU2Nyb2xsQXJlYSgpfSkpKSx0aGlzLl9zZWxlY3Rpb25TZXJ2aWNlPXRoaXMucmVnaXN0ZXIodGhpcy5faW5zdGFudGlhdGlvblNlcnZpY2UuY3JlYXRlSW5zdGFuY2UoZi5TZWxlY3Rpb25TZXJ2aWNlLHRoaXMuZWxlbWVudCx0aGlzLnNjcmVlbkVsZW1lbnQsdGhpcy5saW5raWZpZXIyKSksdGhpcy5faW5zdGFudGlhdGlvblNlcnZpY2Uuc2V0U2VydmljZShFLklTZWxlY3Rpb25TZXJ2aWNlLHRoaXMuX3NlbGVjdGlvblNlcnZpY2UpLHRoaXMucmVnaXN0ZXIodGhpcy5fc2VsZWN0aW9uU2VydmljZS5vblJlcXVlc3RTY3JvbGxMaW5lcygoZnVuY3Rpb24oZSl7cmV0dXJuIHQuc2Nyb2xsTGluZXMoZS5hbW91bnQsZS5zdXBwcmVzc1Njcm9sbEV2ZW50KX0pKSksdGhpcy5yZWdpc3Rlcih0aGlzLl9zZWxlY3Rpb25TZXJ2aWNlLm9uU2VsZWN0aW9uQ2hhbmdlKChmdW5jdGlvbigpe3JldHVybiB0Ll9vblNlbGVjdGlvbkNoYW5nZS5maXJlKCl9KSkpLHRoaXMucmVnaXN0ZXIodGhpcy5fc2VsZWN0aW9uU2VydmljZS5vblJlcXVlc3RSZWRyYXcoKGZ1bmN0aW9uKGUpe3JldHVybiB0Ll9yZW5kZXJTZXJ2aWNlLm9uU2VsZWN0aW9uQ2hhbmdlZChlLnN0YXJ0LGUuZW5kLGUuY29sdW1uU2VsZWN0TW9kZSl9KSkpLHRoaXMucmVnaXN0ZXIodGhpcy5fc2VsZWN0aW9uU2VydmljZS5vbkxpbnV4TW91c2VTZWxlY3Rpb24oKGZ1bmN0aW9uKGUpe3QudGV4dGFyZWEudmFsdWU9ZSx0LnRleHRhcmVhLmZvY3VzKCksdC50ZXh0YXJlYS5zZWxlY3QoKX0pKSksdGhpcy5yZWdpc3Rlcih0aGlzLl9vblNjcm9sbC5ldmVudCgoZnVuY3Rpb24oZSl7dC52aWV3cG9ydC5zeW5jU2Nyb2xsQXJlYSgpLHQuX3NlbGVjdGlvblNlcnZpY2UucmVmcmVzaCgpfSkpKSx0aGlzLnJlZ2lzdGVyKCgwLGQuYWRkRGlzcG9zYWJsZURvbUxpc3RlbmVyKSh0aGlzLl92aWV3cG9ydEVsZW1lbnQsInNjcm9sbCIsKGZ1bmN0aW9uKCl7cmV0dXJuIHQuX3NlbGVjdGlvblNlcnZpY2UucmVmcmVzaCgpfSkpKSx0aGlzLl9tb3VzZVpvbmVNYW5hZ2VyPXRoaXMuX2luc3RhbnRpYXRpb25TZXJ2aWNlLmNyZWF0ZUluc3RhbmNlKGcuTW91c2Vab25lTWFuYWdlcix0aGlzLmVsZW1lbnQsdGhpcy5zY3JlZW5FbGVtZW50KSx0aGlzLnJlZ2lzdGVyKHRoaXMuX21vdXNlWm9uZU1hbmFnZXIpLHRoaXMucmVnaXN0ZXIodGhpcy5vblNjcm9sbCgoZnVuY3Rpb24oKXtyZXR1cm4gdC5fbW91c2Vab25lTWFuYWdlci5jbGVhckFsbCgpfSkpKSx0aGlzLmxpbmtpZmllci5hdHRhY2hUb0RvbSh0aGlzLmVsZW1lbnQsdGhpcy5fbW91c2Vab25lTWFuYWdlciksdGhpcy5saW5raWZpZXIyLmF0dGFjaFRvRG9tKHRoaXMuc2NyZWVuRWxlbWVudCx0aGlzLl9tb3VzZVNlcnZpY2UsdGhpcy5fcmVuZGVyU2VydmljZSksdGhpcy5yZWdpc3RlcigoMCxkLmFkZERpc3Bvc2FibGVEb21MaXN0ZW5lcikodGhpcy5lbGVtZW50LCJtb3VzZWRvd24iLChmdW5jdGlvbihlKXtyZXR1cm4gdC5fc2VsZWN0aW9uU2VydmljZS5vbk1vdXNlRG93bihlKX0pKSksdGhpcy5jb3JlTW91c2VTZXJ2aWNlLmFyZU1vdXNlRXZlbnRzQWN0aXZlPyh0aGlzLl9zZWxlY3Rpb25TZXJ2aWNlLmRpc2FibGUoKSx0aGlzLmVsZW1lbnQuY2xhc3NMaXN0LmFkZCgiZW5hYmxlLW1vdXNlLWV2ZW50cyIpKTp0aGlzLl9zZWxlY3Rpb25TZXJ2aWNlLmVuYWJsZSgpLHRoaXMub3B0aW9ucy5zY3JlZW5SZWFkZXJNb2RlJiYodGhpcy5fYWNjZXNzaWJpbGl0eU1hbmFnZXI9bmV3IHkuQWNjZXNzaWJpbGl0eU1hbmFnZXIodGhpcyx0aGlzLl9yZW5kZXJTZXJ2aWNlKSksdGhpcy5fY2hhclNpemVTZXJ2aWNlLm1lYXN1cmUoKSx0aGlzLnJlZnJlc2goMCx0aGlzLnJvd3MtMSksdGhpcy5faW5pdEdsb2JhbCgpLHRoaXMuYmluZE1vdXNlKCl9LHQucHJvdG90eXBlLl9jcmVhdGVSZW5kZXJlcj1mdW5jdGlvbigpe3N3aXRjaCh0aGlzLm9wdGlvbnMucmVuZGVyZXJUeXBlKXtjYXNlImNhbnZhcyI6cmV0dXJuIHRoaXMuX2luc3RhbnRpYXRpb25TZXJ2aWNlLmNyZWF0ZUluc3RhbmNlKHUuUmVuZGVyZXIsdGhpcy5fY29sb3JNYW5hZ2VyLmNvbG9ycyx0aGlzLnNjcmVlbkVsZW1lbnQsdGhpcy5saW5raWZpZXIsdGhpcy5saW5raWZpZXIyKTtjYXNlImRvbSI6cmV0dXJuIHRoaXMuX2luc3RhbnRpYXRpb25TZXJ2aWNlLmNyZWF0ZUluc3RhbmNlKG0uRG9tUmVuZGVyZXIsdGhpcy5fY29sb3JNYW5hZ2VyLmNvbG9ycyx0aGlzLmVsZW1lbnQsdGhpcy5zY3JlZW5FbGVtZW50LHRoaXMuX3ZpZXdwb3J0RWxlbWVudCx0aGlzLmxpbmtpZmllcix0aGlzLmxpbmtpZmllcjIpO2RlZmF1bHQ6dGhyb3cgbmV3IEVycm9yKCdVbnJlY29nbml6ZWQgcmVuZGVyZXJUeXBlICInK3RoaXMub3B0aW9ucy5yZW5kZXJlclR5cGUrJyInKX19LHQucHJvdG90eXBlLl9zZXRUaGVtZT1mdW5jdGlvbihlKXt2YXIgdCxyLGk7dGhpcy5fdGhlbWU9ZSxudWxsPT09KHQ9dGhpcy5fY29sb3JNYW5hZ2VyKXx8dm9pZCAwPT09dHx8dC5zZXRUaGVtZShlKSxudWxsPT09KHI9dGhpcy5fcmVuZGVyU2VydmljZSl8fHZvaWQgMD09PXJ8fHIuc2V0Q29sb3JzKHRoaXMuX2NvbG9yTWFuYWdlci5jb2xvcnMpLG51bGw9PT0oaT10aGlzLnZpZXdwb3J0KXx8dm9pZCAwPT09aXx8aS5vblRoZW1lQ2hhbmdlKHRoaXMuX2NvbG9yTWFuYWdlci5jb2xvcnMpfSx0LnByb3RvdHlwZS5iaW5kTW91c2U9ZnVuY3Rpb24oKXt2YXIgZT10aGlzLHQ9dGhpcyxyPXRoaXMuZWxlbWVudDtmdW5jdGlvbiBpKGUpe3ZhciByLGksbj10Ll9tb3VzZVNlcnZpY2UuZ2V0UmF3Qnl0ZUNvb3JkcyhlLHQuc2NyZWVuRWxlbWVudCx0LmNvbHMsdC5yb3dzKTtpZighbilyZXR1cm4hMTtzd2l0Y2goZS5vdmVycmlkZVR5cGV8fGUudHlwZSl7Y2FzZSJtb3VzZW1vdmUiOmk9MzIsdm9pZCAwPT09ZS5idXR0b25zPyhyPTMsdm9pZCAwIT09ZS5idXR0b24mJihyPWUuYnV0dG9uPDM/ZS5idXR0b246MykpOnI9MSZlLmJ1dHRvbnM/MDo0JmUuYnV0dG9ucz8xOjImZS5idXR0b25zPzI6MzticmVhaztjYXNlIm1vdXNldXAiOmk9MCxyPWUuYnV0dG9uPDM/ZS5idXR0b246MzticmVhaztjYXNlIm1vdXNlZG93biI6aT0xLHI9ZS5idXR0b248Mz9lLmJ1dHRvbjozO2JyZWFrO2Nhc2Uid2hlZWwiOjAhPT1lLmRlbHRhWSYmKGk9ZS5kZWx0YVk8MD8wOjEpLHI9NDticmVhaztkZWZhdWx0OnJldHVybiExfXJldHVybiEodm9pZCAwPT09aXx8dm9pZCAwPT09cnx8cj40KSYmdC5jb3JlTW91c2VTZXJ2aWNlLnRyaWdnZXJNb3VzZUV2ZW50KHtjb2w6bi54LTMzLHJvdzpuLnktMzMsYnV0dG9uOnIsYWN0aW9uOmksY3RybDplLmN0cmxLZXksYWx0OmUuYWx0S2V5LHNoaWZ0OmUuc2hpZnRLZXl9KX12YXIgbj17bW91c2V1cDpudWxsLHdoZWVsOm51bGwsbW91c2VkcmFnOm51bGwsbW91c2Vtb3ZlOm51bGx9LG89ZnVuY3Rpb24odCl7cmV0dXJuIGkodCksdC5idXR0b25zfHwoZS5fZG9jdW1lbnQucmVtb3ZlRXZlbnRMaXN0ZW5lcigibW91c2V1cCIsbi5tb3VzZXVwKSxuLm1vdXNlZHJhZyYmZS5fZG9jdW1lbnQucmVtb3ZlRXZlbnRMaXN0ZW5lcigibW91c2Vtb3ZlIixuLm1vdXNlZHJhZykpLGUuY2FuY2VsKHQpfSxzPWZ1bmN0aW9uKHQpe3JldHVybiBpKHQpLGUuY2FuY2VsKHQsITApfSxhPWZ1bmN0aW9uKGUpe2UuYnV0dG9ucyYmaShlKX0sbD1mdW5jdGlvbihlKXtlLmJ1dHRvbnN8fGkoZSl9O3RoaXMucmVnaXN0ZXIodGhpcy5jb3JlTW91c2VTZXJ2aWNlLm9uUHJvdG9jb2xDaGFuZ2UoKGZ1bmN0aW9uKHQpe3Q/KCJkZWJ1ZyI9PT1lLm9wdGlvbnNTZXJ2aWNlLm9wdGlvbnMubG9nTGV2ZWwmJmUuX2xvZ1NlcnZpY2UuZGVidWcoIkJpbmRpbmcgdG8gbW91c2UgZXZlbnRzOiIsZS5jb3JlTW91c2VTZXJ2aWNlLmV4cGxhaW5FdmVudHModCkpLGUuZWxlbWVudC5jbGFzc0xpc3QuYWRkKCJlbmFibGUtbW91c2UtZXZlbnRzIiksZS5fc2VsZWN0aW9uU2VydmljZS5kaXNhYmxlKCkpOihlLl9sb2dTZXJ2aWNlLmRlYnVnKCJVbmJpbmRpbmcgZnJvbSBtb3VzZSBldmVudHMuIiksZS5lbGVtZW50LmNsYXNzTGlzdC5yZW1vdmUoImVuYWJsZS1tb3VzZS1ldmVudHMiKSxlLl9zZWxlY3Rpb25TZXJ2aWNlLmVuYWJsZSgpKSw4JnQ/bi5tb3VzZW1vdmV8fChyLmFkZEV2ZW50TGlzdGVuZXIoIm1vdXNlbW92ZSIsbCksbi5tb3VzZW1vdmU9bCk6KHIucmVtb3ZlRXZlbnRMaXN0ZW5lcigibW91c2Vtb3ZlIixuLm1vdXNlbW92ZSksbi5tb3VzZW1vdmU9bnVsbCksMTYmdD9uLndoZWVsfHwoci5hZGRFdmVudExpc3RlbmVyKCJ3aGVlbCIscyx7cGFzc2l2ZTohMX0pLG4ud2hlZWw9cyk6KHIucmVtb3ZlRXZlbnRMaXN0ZW5lcigid2hlZWwiLG4ud2hlZWwpLG4ud2hlZWw9bnVsbCksMiZ0P24ubW91c2V1cHx8KG4ubW91c2V1cD1vKTooZS5fZG9jdW1lbnQucmVtb3ZlRXZlbnRMaXN0ZW5lcigibW91c2V1cCIsbi5tb3VzZXVwKSxuLm1vdXNldXA9bnVsbCksNCZ0P24ubW91c2VkcmFnfHwobi5tb3VzZWRyYWc9YSk6KGUuX2RvY3VtZW50LnJlbW92ZUV2ZW50TGlzdGVuZXIoIm1vdXNlbW92ZSIsbi5tb3VzZWRyYWcpLG4ubW91c2VkcmFnPW51bGwpfSkpKSx0aGlzLmNvcmVNb3VzZVNlcnZpY2UuYWN0aXZlUHJvdG9jb2w9dGhpcy5jb3JlTW91c2VTZXJ2aWNlLmFjdGl2ZVByb3RvY29sLHRoaXMucmVnaXN0ZXIoKDAsZC5hZGREaXNwb3NhYmxlRG9tTGlzdGVuZXIpKHIsIm1vdXNlZG93biIsKGZ1bmN0aW9uKHQpe2lmKHQucHJldmVudERlZmF1bHQoKSxlLmZvY3VzKCksZS5jb3JlTW91c2VTZXJ2aWNlLmFyZU1vdXNlRXZlbnRzQWN0aXZlJiYhZS5fc2VsZWN0aW9uU2VydmljZS5zaG91bGRGb3JjZVNlbGVjdGlvbih0KSlyZXR1cm4gaSh0KSxuLm1vdXNldXAmJmUuX2RvY3VtZW50LmFkZEV2ZW50TGlzdGVuZXIoIm1vdXNldXAiLG4ubW91c2V1cCksbi5tb3VzZWRyYWcmJmUuX2RvY3VtZW50LmFkZEV2ZW50TGlzdGVuZXIoIm1vdXNlbW92ZSIsbi5tb3VzZWRyYWcpLGUuY2FuY2VsKHQpfSkpKSx0aGlzLnJlZ2lzdGVyKCgwLGQuYWRkRGlzcG9zYWJsZURvbUxpc3RlbmVyKShyLCJ3aGVlbCIsKGZ1bmN0aW9uKHQpe2lmKCFuLndoZWVsKXtpZighZS5idWZmZXIuaGFzU2Nyb2xsYmFjayl7dmFyIHI9ZS52aWV3cG9ydC5nZXRMaW5lc1Njcm9sbGVkKHQpO2lmKDA9PT1yKXJldHVybjtmb3IodmFyIGk9Yy5DMC5FU0MrKGUuY29yZVNlcnZpY2UuZGVjUHJpdmF0ZU1vZGVzLmFwcGxpY2F0aW9uQ3Vyc29yS2V5cz8iTyI6IlsiKSsodC5kZWx0YVk8MD8iQSI6IkIiKSxvPSIiLHM9MDtzPE1hdGguYWJzKHIpO3MrKylvKz1pO3JldHVybiBlLmNvcmVTZXJ2aWNlLnRyaWdnZXJEYXRhRXZlbnQobywhMCksZS5jYW5jZWwodCwhMCl9cmV0dXJuIGUudmlld3BvcnQub25XaGVlbCh0KT9lLmNhbmNlbCh0KTp2b2lkIDB9fSkse3Bhc3NpdmU6ITF9KSksdGhpcy5yZWdpc3RlcigoMCxkLmFkZERpc3Bvc2FibGVEb21MaXN0ZW5lcikociwidG91Y2hzdGFydCIsKGZ1bmN0aW9uKHQpe2lmKCFlLmNvcmVNb3VzZVNlcnZpY2UuYXJlTW91c2VFdmVudHNBY3RpdmUpcmV0dXJuIGUudmlld3BvcnQub25Ub3VjaFN0YXJ0KHQpLGUuY2FuY2VsKHQpfSkse3Bhc3NpdmU6ITB9KSksdGhpcy5yZWdpc3RlcigoMCxkLmFkZERpc3Bvc2FibGVEb21MaXN0ZW5lcikociwidG91Y2htb3ZlIiwoZnVuY3Rpb24odCl7aWYoIWUuY29yZU1vdXNlU2VydmljZS5hcmVNb3VzZUV2ZW50c0FjdGl2ZSlyZXR1cm4gZS52aWV3cG9ydC5vblRvdWNoTW92ZSh0KT92b2lkIDA6ZS5jYW5jZWwodCl9KSx7cGFzc2l2ZTohMX0pKX0sdC5wcm90b3R5cGUucmVmcmVzaD1mdW5jdGlvbihlLHQpe3ZhciByO251bGw9PT0ocj10aGlzLl9yZW5kZXJTZXJ2aWNlKXx8dm9pZCAwPT09cnx8ci5yZWZyZXNoUm93cyhlLHQpfSx0LnByb3RvdHlwZS5fcXVldWVMaW5raWZpY2F0aW9uPWZ1bmN0aW9uKGUsdCl7dmFyIHI7bnVsbD09PShyPXRoaXMubGlua2lmaWVyKXx8dm9pZCAwPT09cnx8ci5saW5raWZ5Um93cyhlLHQpfSx0LnByb3RvdHlwZS51cGRhdGVDdXJzb3JTdHlsZT1mdW5jdGlvbihlKXt2YXIgdDsobnVsbD09PSh0PXRoaXMuX3NlbGVjdGlvblNlcnZpY2UpfHx2b2lkIDA9PT10P3ZvaWQgMDp0LnNob3VsZENvbHVtblNlbGVjdChlKSk/dGhpcy5lbGVtZW50LmNsYXNzTGlzdC5hZGQoImNvbHVtbi1zZWxlY3QiKTp0aGlzLmVsZW1lbnQuY2xhc3NMaXN0LnJlbW92ZSgiY29sdW1uLXNlbGVjdCIpfSx0LnByb3RvdHlwZS5fc2hvd0N1cnNvcj1mdW5jdGlvbigpe3RoaXMuY29yZVNlcnZpY2UuaXNDdXJzb3JJbml0aWFsaXplZHx8KHRoaXMuY29yZVNlcnZpY2UuaXNDdXJzb3JJbml0aWFsaXplZD0hMCx0aGlzLnJlZnJlc2godGhpcy5idWZmZXIueSx0aGlzLmJ1ZmZlci55KSl9LHQucHJvdG90eXBlLnNjcm9sbExpbmVzPWZ1bmN0aW9uKHQscixpKXt2b2lkIDA9PT1pJiYoaT0wKSxlLnByb3RvdHlwZS5zY3JvbGxMaW5lcy5jYWxsKHRoaXMsdCxyLGkpLHRoaXMucmVmcmVzaCgwLHRoaXMucm93cy0xKX0sdC5wcm90b3R5cGUucGFzdGU9ZnVuY3Rpb24oZSl7KDAsYS5wYXN0ZSkoZSx0aGlzLnRleHRhcmVhLHRoaXMuY29yZVNlcnZpY2UpfSx0LnByb3RvdHlwZS5hdHRhY2hDdXN0b21LZXlFdmVudEhhbmRsZXI9ZnVuY3Rpb24oZSl7dGhpcy5fY3VzdG9tS2V5RXZlbnRIYW5kbGVyPWV9LHQucHJvdG90eXBlLnJlZ2lzdGVyTGlua01hdGNoZXI9ZnVuY3Rpb24oZSx0LHIpe3ZhciBpPXRoaXMubGlua2lmaWVyLnJlZ2lzdGVyTGlua01hdGNoZXIoZSx0LHIpO3JldHVybiB0aGlzLnJlZnJlc2goMCx0aGlzLnJvd3MtMSksaX0sdC5wcm90b3R5cGUuZGVyZWdpc3RlckxpbmtNYXRjaGVyPWZ1bmN0aW9uKGUpe3RoaXMubGlua2lmaWVyLmRlcmVnaXN0ZXJMaW5rTWF0Y2hlcihlKSYmdGhpcy5yZWZyZXNoKDAsdGhpcy5yb3dzLTEpfSx0LnByb3RvdHlwZS5yZWdpc3RlckxpbmtQcm92aWRlcj1mdW5jdGlvbihlKXtyZXR1cm4gdGhpcy5saW5raWZpZXIyLnJlZ2lzdGVyTGlua1Byb3ZpZGVyKGUpfSx0LnByb3RvdHlwZS5yZWdpc3RlckNoYXJhY3RlckpvaW5lcj1mdW5jdGlvbihlKXtpZighdGhpcy5fY2hhcmFjdGVySm9pbmVyU2VydmljZSl0aHJvdyBuZXcgRXJyb3IoIlRlcm1pbmFsIG11c3QgYmUgb3BlbmVkIGZpcnN0Iik7dmFyIHQ9dGhpcy5fY2hhcmFjdGVySm9pbmVyU2VydmljZS5yZWdpc3RlcihlKTtyZXR1cm4gdGhpcy5yZWZyZXNoKDAsdGhpcy5yb3dzLTEpLHR9LHQucHJvdG90eXBlLmRlcmVnaXN0ZXJDaGFyYWN0ZXJKb2luZXI9ZnVuY3Rpb24oZSl7aWYoIXRoaXMuX2NoYXJhY3RlckpvaW5lclNlcnZpY2UpdGhyb3cgbmV3IEVycm9yKCJUZXJtaW5hbCBtdXN0IGJlIG9wZW5lZCBmaXJzdCIpO3RoaXMuX2NoYXJhY3RlckpvaW5lclNlcnZpY2UuZGVyZWdpc3RlcihlKSYmdGhpcy5yZWZyZXNoKDAsdGhpcy5yb3dzLTEpfSxPYmplY3QuZGVmaW5lUHJvcGVydHkodC5wcm90b3R5cGUsIm1hcmtlcnMiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5idWZmZXIubWFya2Vyc30sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSx0LnByb3RvdHlwZS5hZGRNYXJrZXI9ZnVuY3Rpb24oZSl7aWYodGhpcy5idWZmZXI9PT10aGlzLmJ1ZmZlcnMubm9ybWFsKXJldHVybiB0aGlzLmJ1ZmZlci5hZGRNYXJrZXIodGhpcy5idWZmZXIueWJhc2UrdGhpcy5idWZmZXIueStlKX0sdC5wcm90b3R5cGUuaGFzU2VsZWN0aW9uPWZ1bmN0aW9uKCl7cmV0dXJuISF0aGlzLl9zZWxlY3Rpb25TZXJ2aWNlJiZ0aGlzLl9zZWxlY3Rpb25TZXJ2aWNlLmhhc1NlbGVjdGlvbn0sdC5wcm90b3R5cGUuc2VsZWN0PWZ1bmN0aW9uKGUsdCxyKXt0aGlzLl9zZWxlY3Rpb25TZXJ2aWNlLnNldFNlbGVjdGlvbihlLHQscil9LHQucHJvdG90eXBlLmdldFNlbGVjdGlvbj1mdW5jdGlvbigpe3JldHVybiB0aGlzLl9zZWxlY3Rpb25TZXJ2aWNlP3RoaXMuX3NlbGVjdGlvblNlcnZpY2Uuc2VsZWN0aW9uVGV4dDoiIn0sdC5wcm90b3R5cGUuZ2V0U2VsZWN0aW9uUG9zaXRpb249ZnVuY3Rpb24oKXtpZih0aGlzLl9zZWxlY3Rpb25TZXJ2aWNlJiZ0aGlzLl9zZWxlY3Rpb25TZXJ2aWNlLmhhc1NlbGVjdGlvbilyZXR1cm57c3RhcnRDb2x1bW46dGhpcy5fc2VsZWN0aW9uU2VydmljZS5zZWxlY3Rpb25TdGFydFswXSxzdGFydFJvdzp0aGlzLl9zZWxlY3Rpb25TZXJ2aWNlLnNlbGVjdGlvblN0YXJ0WzFdLGVuZENvbHVtbjp0aGlzLl9zZWxlY3Rpb25TZXJ2aWNlLnNlbGVjdGlvbkVuZFswXSxlbmRSb3c6dGhpcy5fc2VsZWN0aW9uU2VydmljZS5zZWxlY3Rpb25FbmRbMV19fSx0LnByb3RvdHlwZS5jbGVhclNlbGVjdGlvbj1mdW5jdGlvbigpe3ZhciBlO251bGw9PT0oZT10aGlzLl9zZWxlY3Rpb25TZXJ2aWNlKXx8dm9pZCAwPT09ZXx8ZS5jbGVhclNlbGVjdGlvbigpfSx0LnByb3RvdHlwZS5zZWxlY3RBbGw9ZnVuY3Rpb24oKXt2YXIgZTtudWxsPT09KGU9dGhpcy5fc2VsZWN0aW9uU2VydmljZSl8fHZvaWQgMD09PWV8fGUuc2VsZWN0QWxsKCl9LHQucHJvdG90eXBlLnNlbGVjdExpbmVzPWZ1bmN0aW9uKGUsdCl7dmFyIHI7bnVsbD09PShyPXRoaXMuX3NlbGVjdGlvblNlcnZpY2UpfHx2b2lkIDA9PT1yfHxyLnNlbGVjdExpbmVzKGUsdCl9LHQucHJvdG90eXBlLl9rZXlEb3duPWZ1bmN0aW9uKGUpe2lmKHRoaXMuX2tleURvd25IYW5kbGVkPSExLHRoaXMuX2N1c3RvbUtleUV2ZW50SGFuZGxlciYmITE9PT10aGlzLl9jdXN0b21LZXlFdmVudEhhbmRsZXIoZSkpcmV0dXJuITE7aWYoIXRoaXMuX2NvbXBvc2l0aW9uSGVscGVyLmtleWRvd24oZSkpcmV0dXJuIHRoaXMuYnVmZmVyLnliYXNlIT09dGhpcy5idWZmZXIueWRpc3AmJnRoaXMuX2J1ZmZlclNlcnZpY2Uuc2Nyb2xsVG9Cb3R0b20oKSwhMTsiRGVhZCIhPT1lLmtleSYmIkFsdEdyYXBoIiE9PWUua2V5fHwodGhpcy5fdW5wcm9jZXNzZWREZWFkS2V5PSEwKTt2YXIgdD0oMCxiLmV2YWx1YXRlS2V5Ym9hcmRFdmVudCkoZSx0aGlzLmNvcmVTZXJ2aWNlLmRlY1ByaXZhdGVNb2Rlcy5hcHBsaWNhdGlvbkN1cnNvcktleXMsdGhpcy5icm93c2VyLmlzTWFjLHRoaXMub3B0aW9ucy5tYWNPcHRpb25Jc01ldGEpO2lmKHRoaXMudXBkYXRlQ3Vyc29yU3R5bGUoZSksMz09PXQudHlwZXx8Mj09PXQudHlwZSl7dmFyIHI9dGhpcy5yb3dzLTE7cmV0dXJuIHRoaXMuc2Nyb2xsTGluZXMoMj09PXQudHlwZT8tcjpyKSx0aGlzLmNhbmNlbChlLCEwKX1yZXR1cm4gMT09PXQudHlwZSYmdGhpcy5zZWxlY3RBbGwoKSwhIXRoaXMuX2lzVGhpcmRMZXZlbFNoaWZ0KHRoaXMuYnJvd3NlcixlKXx8KHQuY2FuY2VsJiZ0aGlzLmNhbmNlbChlLCEwKSwhdC5rZXl8fCh0aGlzLl91bnByb2Nlc3NlZERlYWRLZXk/KHRoaXMuX3VucHJvY2Vzc2VkRGVhZEtleT0hMSwhMCk6KHQua2V5IT09Yy5DMC5FVFgmJnQua2V5IT09Yy5DMC5DUnx8KHRoaXMudGV4dGFyZWEudmFsdWU9IiIpLHRoaXMuX29uS2V5LmZpcmUoe2tleTp0LmtleSxkb21FdmVudDplfSksdGhpcy5fc2hvd0N1cnNvcigpLHRoaXMuY29yZVNlcnZpY2UudHJpZ2dlckRhdGFFdmVudCh0LmtleSwhMCksdGhpcy5vcHRpb25zU2VydmljZS5vcHRpb25zLnNjcmVlblJlYWRlck1vZGU/dm9pZCh0aGlzLl9rZXlEb3duSGFuZGxlZD0hMCk6dGhpcy5jYW5jZWwoZSwhMCkpKSl9LHQucHJvdG90eXBlLl9pc1RoaXJkTGV2ZWxTaGlmdD1mdW5jdGlvbihlLHQpe3ZhciByPWUuaXNNYWMmJiF0aGlzLm9wdGlvbnMubWFjT3B0aW9uSXNNZXRhJiZ0LmFsdEtleSYmIXQuY3RybEtleSYmIXQubWV0YUtleXx8ZS5pc1dpbmRvd3MmJnQuYWx0S2V5JiZ0LmN0cmxLZXkmJiF0Lm1ldGFLZXl8fGUuaXNXaW5kb3dzJiZ0LmdldE1vZGlmaWVyU3RhdGUoIkFsdEdyYXBoIik7cmV0dXJuImtleXByZXNzIj09PXQudHlwZT9yOnImJighdC5rZXlDb2RlfHx0LmtleUNvZGU+NDcpfSx0LnByb3RvdHlwZS5fa2V5VXA9ZnVuY3Rpb24oZSl7dGhpcy5fY3VzdG9tS2V5RXZlbnRIYW5kbGVyJiYhMT09PXRoaXMuX2N1c3RvbUtleUV2ZW50SGFuZGxlcihlKXx8KGZ1bmN0aW9uKGUpe3JldHVybiAxNj09PWUua2V5Q29kZXx8MTc9PT1lLmtleUNvZGV8fDE4PT09ZS5rZXlDb2RlfShlKXx8dGhpcy5mb2N1cygpLHRoaXMudXBkYXRlQ3Vyc29yU3R5bGUoZSksdGhpcy5fa2V5UHJlc3NIYW5kbGVkPSExKX0sdC5wcm90b3R5cGUuX2tleVByZXNzPWZ1bmN0aW9uKGUpe3ZhciB0O2lmKHRoaXMuX2tleVByZXNzSGFuZGxlZD0hMSx0aGlzLl9rZXlEb3duSGFuZGxlZClyZXR1cm4hMTtpZih0aGlzLl9jdXN0b21LZXlFdmVudEhhbmRsZXImJiExPT09dGhpcy5fY3VzdG9tS2V5RXZlbnRIYW5kbGVyKGUpKXJldHVybiExO2lmKHRoaXMuY2FuY2VsKGUpLGUuY2hhckNvZGUpdD1lLmNoYXJDb2RlO2Vsc2UgaWYobnVsbD09PWUud2hpY2h8fHZvaWQgMD09PWUud2hpY2gpdD1lLmtleUNvZGU7ZWxzZXtpZigwPT09ZS53aGljaHx8MD09PWUuY2hhckNvZGUpcmV0dXJuITE7dD1lLndoaWNofXJldHVybiEoIXR8fChlLmFsdEtleXx8ZS5jdHJsS2V5fHxlLm1ldGFLZXkpJiYhdGhpcy5faXNUaGlyZExldmVsU2hpZnQodGhpcy5icm93c2VyLGUpfHwodD1TdHJpbmcuZnJvbUNoYXJDb2RlKHQpLHRoaXMuX29uS2V5LmZpcmUoe2tleTp0LGRvbUV2ZW50OmV9KSx0aGlzLl9zaG93Q3Vyc29yKCksdGhpcy5jb3JlU2VydmljZS50cmlnZ2VyRGF0YUV2ZW50KHQsITApLHRoaXMuX2tleVByZXNzSGFuZGxlZD0hMCx0aGlzLl91bnByb2Nlc3NlZERlYWRLZXk9ITEsMCkpfSx0LnByb3RvdHlwZS5faW5wdXRFdmVudD1mdW5jdGlvbihlKXtpZihlLmRhdGEmJiJpbnNlcnRUZXh0Ij09PWUuaW5wdXRUeXBlJiYhZS5jb21wb3NlZCYmIXRoaXMub3B0aW9uc1NlcnZpY2Uub3B0aW9ucy5zY3JlZW5SZWFkZXJNb2RlKXtpZih0aGlzLl9rZXlQcmVzc0hhbmRsZWQpcmV0dXJuITE7dGhpcy5fdW5wcm9jZXNzZWREZWFkS2V5PSExO3ZhciB0PWUuZGF0YTtyZXR1cm4gdGhpcy5jb3JlU2VydmljZS50cmlnZ2VyRGF0YUV2ZW50KHQsITApLHRoaXMuY2FuY2VsKGUpLCEwfXJldHVybiExfSx0LnByb3RvdHlwZS5iZWxsPWZ1bmN0aW9uKCl7dmFyIGU7dGhpcy5fc291bmRCZWxsKCkmJihudWxsPT09KGU9dGhpcy5fc291bmRTZXJ2aWNlKXx8dm9pZCAwPT09ZXx8ZS5wbGF5QmVsbFNvdW5kKCkpLHRoaXMuX29uQmVsbC5maXJlKCl9LHQucHJvdG90eXBlLnJlc2l6ZT1mdW5jdGlvbih0LHIpe3QhPT10aGlzLmNvbHN8fHIhPT10aGlzLnJvd3M/ZS5wcm90b3R5cGUucmVzaXplLmNhbGwodGhpcyx0LHIpOnRoaXMuX2NoYXJTaXplU2VydmljZSYmIXRoaXMuX2NoYXJTaXplU2VydmljZS5oYXNWYWxpZFNpemUmJnRoaXMuX2NoYXJTaXplU2VydmljZS5tZWFzdXJlKCl9LHQucHJvdG90eXBlLl9hZnRlclJlc2l6ZT1mdW5jdGlvbihlLHQpe3ZhciByLGk7bnVsbD09PShyPXRoaXMuX2NoYXJTaXplU2VydmljZSl8fHZvaWQgMD09PXJ8fHIubWVhc3VyZSgpLG51bGw9PT0oaT10aGlzLnZpZXdwb3J0KXx8dm9pZCAwPT09aXx8aS5zeW5jU2Nyb2xsQXJlYSghMCl9LHQucHJvdG90eXBlLmNsZWFyPWZ1bmN0aW9uKCl7aWYoMCE9PXRoaXMuYnVmZmVyLnliYXNlfHwwIT09dGhpcy5idWZmZXIueSl7dGhpcy5idWZmZXIubGluZXMuc2V0KDAsdGhpcy5idWZmZXIubGluZXMuZ2V0KHRoaXMuYnVmZmVyLnliYXNlK3RoaXMuYnVmZmVyLnkpKSx0aGlzLmJ1ZmZlci5saW5lcy5sZW5ndGg9MSx0aGlzLmJ1ZmZlci55ZGlzcD0wLHRoaXMuYnVmZmVyLnliYXNlPTAsdGhpcy5idWZmZXIueT0wO2Zvcih2YXIgZT0xO2U8dGhpcy5yb3dzO2UrKyl0aGlzLmJ1ZmZlci5saW5lcy5wdXNoKHRoaXMuYnVmZmVyLmdldEJsYW5rTGluZShDLkRFRkFVTFRfQVRUUl9EQVRBKSk7dGhpcy5yZWZyZXNoKDAsdGhpcy5yb3dzLTEpLHRoaXMuX29uU2Nyb2xsLmZpcmUoe3Bvc2l0aW9uOnRoaXMuYnVmZmVyLnlkaXNwLHNvdXJjZTowfSl9fSx0LnByb3RvdHlwZS5yZXNldD1mdW5jdGlvbigpe3ZhciB0LHI7dGhpcy5vcHRpb25zLnJvd3M9dGhpcy5yb3dzLHRoaXMub3B0aW9ucy5jb2xzPXRoaXMuY29sczt2YXIgaT10aGlzLl9jdXN0b21LZXlFdmVudEhhbmRsZXI7dGhpcy5fc2V0dXAoKSxlLnByb3RvdHlwZS5yZXNldC5jYWxsKHRoaXMpLG51bGw9PT0odD10aGlzLl9zZWxlY3Rpb25TZXJ2aWNlKXx8dm9pZCAwPT09dHx8dC5yZXNldCgpLHRoaXMuX2N1c3RvbUtleUV2ZW50SGFuZGxlcj1pLHRoaXMucmVmcmVzaCgwLHRoaXMucm93cy0xKSxudWxsPT09KHI9dGhpcy52aWV3cG9ydCl8fHZvaWQgMD09PXJ8fHIuc3luY1Njcm9sbEFyZWEoKX0sdC5wcm90b3R5cGUuY2xlYXJUZXh0dXJlQXRsYXM9ZnVuY3Rpb24oKXt2YXIgZTtudWxsPT09KGU9dGhpcy5fcmVuZGVyU2VydmljZSl8fHZvaWQgMD09PWV8fGUuY2xlYXJUZXh0dXJlQXRsYXMoKX0sdC5wcm90b3R5cGUuX3JlcG9ydEZvY3VzPWZ1bmN0aW9uKCl7dmFyIGU7KG51bGw9PT0oZT10aGlzLmVsZW1lbnQpfHx2b2lkIDA9PT1lP3ZvaWQgMDplLmNsYXNzTGlzdC5jb250YWlucygiZm9jdXMiKSk/dGhpcy5jb3JlU2VydmljZS50cmlnZ2VyRGF0YUV2ZW50KGMuQzAuRVNDKyJbSSIpOnRoaXMuY29yZVNlcnZpY2UudHJpZ2dlckRhdGFFdmVudChjLkMwLkVTQysiW08iKX0sdC5wcm90b3R5cGUuX3JlcG9ydFdpbmRvd3NPcHRpb25zPWZ1bmN0aW9uKGUpe2lmKHRoaXMuX3JlbmRlclNlcnZpY2Upc3dpdGNoKGUpe2Nhc2UgbC5XaW5kb3dzT3B0aW9uc1JlcG9ydFR5cGUuR0VUX1dJTl9TSVpFX1BJWEVMUzp2YXIgdD10aGlzLl9yZW5kZXJTZXJ2aWNlLmRpbWVuc2lvbnMuc2NhbGVkQ2FudmFzV2lkdGgudG9GaXhlZCgwKSxyPXRoaXMuX3JlbmRlclNlcnZpY2UuZGltZW5zaW9ucy5zY2FsZWRDYW52YXNIZWlnaHQudG9GaXhlZCgwKTt0aGlzLmNvcmVTZXJ2aWNlLnRyaWdnZXJEYXRhRXZlbnQoYy5DMC5FU0MrIls0OyIrcisiOyIrdCsidCIpO2JyZWFrO2Nhc2UgbC5XaW5kb3dzT3B0aW9uc1JlcG9ydFR5cGUuR0VUX0NFTExfU0laRV9QSVhFTFM6dmFyIGk9dGhpcy5fcmVuZGVyU2VydmljZS5kaW1lbnNpb25zLnNjYWxlZENlbGxXaWR0aC50b0ZpeGVkKDApLG49dGhpcy5fcmVuZGVyU2VydmljZS5kaW1lbnNpb25zLnNjYWxlZENlbGxIZWlnaHQudG9GaXhlZCgwKTt0aGlzLmNvcmVTZXJ2aWNlLnRyaWdnZXJEYXRhRXZlbnQoYy5DMC5FU0MrIls2OyIrbisiOyIraSsidCIpfX0sdC5wcm90b3R5cGUuY2FuY2VsPWZ1bmN0aW9uKGUsdCl7aWYodGhpcy5vcHRpb25zLmNhbmNlbEV2ZW50c3x8dClyZXR1cm4gZS5wcmV2ZW50RGVmYXVsdCgpLGUuc3RvcFByb3BhZ2F0aW9uKCksITF9LHQucHJvdG90eXBlLl92aXN1YWxCZWxsPWZ1bmN0aW9uKCl7cmV0dXJuITF9LHQucHJvdG90eXBlLl9zb3VuZEJlbGw9ZnVuY3Rpb24oKXtyZXR1cm4ic291bmQiPT09dGhpcy5vcHRpb25zLmJlbGxTdHlsZX0sdH0oUi5Db3JlVGVybWluYWwpO3QuVGVybWluYWw9UH0sOTkyNDooZSx0KT0+e09iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LCJfX2VzTW9kdWxlIix7dmFsdWU6ITB9KSx0LlRpbWVCYXNlZERlYm91bmNlcj12b2lkIDA7dmFyIHI9ZnVuY3Rpb24oKXtmdW5jdGlvbiBlKGUsdCl7dm9pZCAwPT09dCYmKHQ9MWUzKSx0aGlzLl9yZW5kZXJDYWxsYmFjaz1lLHRoaXMuX2RlYm91bmNlVGhyZXNob2xkTVM9dCx0aGlzLl9sYXN0UmVmcmVzaE1zPTAsdGhpcy5fYWRkaXRpb25hbFJlZnJlc2hSZXF1ZXN0ZWQ9ITF9cmV0dXJuIGUucHJvdG90eXBlLmRpc3Bvc2U9ZnVuY3Rpb24oKXt0aGlzLl9yZWZyZXNoVGltZW91dElEJiZjbGVhclRpbWVvdXQodGhpcy5fcmVmcmVzaFRpbWVvdXRJRCl9LGUucHJvdG90eXBlLnJlZnJlc2g9ZnVuY3Rpb24oZSx0LHIpe3ZhciBpPXRoaXM7dGhpcy5fcm93Q291bnQ9cixlPXZvaWQgMCE9PWU/ZTowLHQ9dm9pZCAwIT09dD90OnRoaXMuX3Jvd0NvdW50LTEsdGhpcy5fcm93U3RhcnQ9dm9pZCAwIT09dGhpcy5fcm93U3RhcnQ/TWF0aC5taW4odGhpcy5fcm93U3RhcnQsZSk6ZSx0aGlzLl9yb3dFbmQ9dm9pZCAwIT09dGhpcy5fcm93RW5kP01hdGgubWF4KHRoaXMuX3Jvd0VuZCx0KTp0O3ZhciBuPURhdGUubm93KCk7aWYobi10aGlzLl9sYXN0UmVmcmVzaE1zPj10aGlzLl9kZWJvdW5jZVRocmVzaG9sZE1TKXRoaXMuX2xhc3RSZWZyZXNoTXM9bix0aGlzLl9pbm5lclJlZnJlc2goKTtlbHNlIGlmKCF0aGlzLl9hZGRpdGlvbmFsUmVmcmVzaFJlcXVlc3RlZCl7dmFyIG89bi10aGlzLl9sYXN0UmVmcmVzaE1zLHM9dGhpcy5fZGVib3VuY2VUaHJlc2hvbGRNUy1vO3RoaXMuX2FkZGl0aW9uYWxSZWZyZXNoUmVxdWVzdGVkPSEwLHRoaXMuX3JlZnJlc2hUaW1lb3V0SUQ9d2luZG93LnNldFRpbWVvdXQoKGZ1bmN0aW9uKCl7aS5fbGFzdFJlZnJlc2hNcz1EYXRlLm5vdygpLGkuX2lubmVyUmVmcmVzaCgpLGkuX2FkZGl0aW9uYWxSZWZyZXNoUmVxdWVzdGVkPSExLGkuX3JlZnJlc2hUaW1lb3V0SUQ9dm9pZCAwfSkscyl9fSxlLnByb3RvdHlwZS5faW5uZXJSZWZyZXNoPWZ1bmN0aW9uKCl7aWYodm9pZCAwIT09dGhpcy5fcm93U3RhcnQmJnZvaWQgMCE9PXRoaXMuX3Jvd0VuZCYmdm9pZCAwIT09dGhpcy5fcm93Q291bnQpe3ZhciBlPU1hdGgubWF4KHRoaXMuX3Jvd1N0YXJ0LDApLHQ9TWF0aC5taW4odGhpcy5fcm93RW5kLHRoaXMuX3Jvd0NvdW50LTEpO3RoaXMuX3Jvd1N0YXJ0PXZvaWQgMCx0aGlzLl9yb3dFbmQ9dm9pZCAwLHRoaXMuX3JlbmRlckNhbGxiYWNrKGUsdCl9fSxlfSgpO3QuVGltZUJhc2VkRGVib3VuY2VyPXJ9LDE2ODA6ZnVuY3Rpb24oZSx0LHIpe3ZhciBpLG49dGhpcyYmdGhpcy5fX2V4dGVuZHN8fChpPWZ1bmN0aW9uKGUsdCl7cmV0dXJuIGk9T2JqZWN0LnNldFByb3RvdHlwZU9mfHx7X19wcm90b19fOltdfWluc3RhbmNlb2YgQXJyYXkmJmZ1bmN0aW9uKGUsdCl7ZS5fX3Byb3RvX189dH18fGZ1bmN0aW9uKGUsdCl7Zm9yKHZhciByIGluIHQpT2JqZWN0LnByb3RvdHlwZS5oYXNPd25Qcm9wZXJ0eS5jYWxsKHQscikmJihlW3JdPXRbcl0pfSxpKGUsdCl9LGZ1bmN0aW9uKGUsdCl7aWYoImZ1bmN0aW9uIiE9dHlwZW9mIHQmJm51bGwhPT10KXRocm93IG5ldyBUeXBlRXJyb3IoIkNsYXNzIGV4dGVuZHMgdmFsdWUgIitTdHJpbmcodCkrIiBpcyBub3QgYSBjb25zdHJ1Y3RvciBvciBudWxsIik7ZnVuY3Rpb24gcigpe3RoaXMuY29uc3RydWN0b3I9ZX1pKGUsdCksZS5wcm90b3R5cGU9bnVsbD09PXQ/T2JqZWN0LmNyZWF0ZSh0KTooci5wcm90b3R5cGU9dC5wcm90b3R5cGUsbmV3IHIpfSksbz10aGlzJiZ0aGlzLl9fZGVjb3JhdGV8fGZ1bmN0aW9uKGUsdCxyLGkpe3ZhciBuLG89YXJndW1lbnRzLmxlbmd0aCxzPW88Mz90Om51bGw9PT1pP2k9T2JqZWN0LmdldE93blByb3BlcnR5RGVzY3JpcHRvcih0LHIpOmk7aWYoIm9iamVjdCI9PXR5cGVvZiBSZWZsZWN0JiYiZnVuY3Rpb24iPT10eXBlb2YgUmVmbGVjdC5kZWNvcmF0ZSlzPVJlZmxlY3QuZGVjb3JhdGUoZSx0LHIsaSk7ZWxzZSBmb3IodmFyIGE9ZS5sZW5ndGgtMTthPj0wO2EtLSkobj1lW2FdKSYmKHM9KG88Mz9uKHMpOm8+Mz9uKHQscixzKTpuKHQscikpfHxzKTtyZXR1cm4gbz4zJiZzJiZPYmplY3QuZGVmaW5lUHJvcGVydHkodCxyLHMpLHN9LHM9dGhpcyYmdGhpcy5fX3BhcmFtfHxmdW5jdGlvbihlLHQpe3JldHVybiBmdW5jdGlvbihyLGkpe3QocixpLGUpfX07T2JqZWN0LmRlZmluZVByb3BlcnR5KHQsIl9fZXNNb2R1bGUiLHt2YWx1ZTohMH0pLHQuVmlld3BvcnQ9dm9pZCAwO3ZhciBhPXIoODQ0KSxjPXIoMzY1NiksbD1yKDQ3MjUpLHU9cigyNTg1KSxoPWZ1bmN0aW9uKGUpe2Z1bmN0aW9uIHQodCxyLGksbixvLHMsYSxsKXt2YXIgdT1lLmNhbGwodGhpcyl8fHRoaXM7cmV0dXJuIHUuX3Njcm9sbExpbmVzPXQsdS5fdmlld3BvcnRFbGVtZW50PXIsdS5fc2Nyb2xsQXJlYT1pLHUuX2VsZW1lbnQ9bix1Ll9idWZmZXJTZXJ2aWNlPW8sdS5fb3B0aW9uc1NlcnZpY2U9cyx1Ll9jaGFyU2l6ZVNlcnZpY2U9YSx1Ll9yZW5kZXJTZXJ2aWNlPWwsdS5zY3JvbGxCYXJXaWR0aD0wLHUuX2N1cnJlbnRSb3dIZWlnaHQ9MCx1Ll9jdXJyZW50U2NhbGVkQ2VsbEhlaWdodD0wLHUuX2xhc3RSZWNvcmRlZEJ1ZmZlckxlbmd0aD0wLHUuX2xhc3RSZWNvcmRlZFZpZXdwb3J0SGVpZ2h0PTAsdS5fbGFzdFJlY29yZGVkQnVmZmVySGVpZ2h0PTAsdS5fbGFzdFRvdWNoWT0wLHUuX2xhc3RTY3JvbGxUb3A9MCx1Ll9sYXN0SGFkU2Nyb2xsQmFyPSExLHUuX3doZWVsUGFydGlhbFNjcm9sbD0wLHUuX3JlZnJlc2hBbmltYXRpb25GcmFtZT1udWxsLHUuX2lnbm9yZU5leHRTY3JvbGxFdmVudD0hMSx1LnNjcm9sbEJhcldpZHRoPXUuX3ZpZXdwb3J0RWxlbWVudC5vZmZzZXRXaWR0aC11Ll9zY3JvbGxBcmVhLm9mZnNldFdpZHRofHwxNSx1Ll9sYXN0SGFkU2Nyb2xsQmFyPSEwLHUucmVnaXN0ZXIoKDAsYy5hZGREaXNwb3NhYmxlRG9tTGlzdGVuZXIpKHUuX3ZpZXdwb3J0RWxlbWVudCwic2Nyb2xsIix1Ll9vblNjcm9sbC5iaW5kKHUpKSksdS5fYWN0aXZlQnVmZmVyPXUuX2J1ZmZlclNlcnZpY2UuYnVmZmVyLHUucmVnaXN0ZXIodS5fYnVmZmVyU2VydmljZS5idWZmZXJzLm9uQnVmZmVyQWN0aXZhdGUoKGZ1bmN0aW9uKGUpe3JldHVybiB1Ll9hY3RpdmVCdWZmZXI9ZS5hY3RpdmVCdWZmZXJ9KSkpLHUuX3JlbmRlckRpbWVuc2lvbnM9dS5fcmVuZGVyU2VydmljZS5kaW1lbnNpb25zLHUucmVnaXN0ZXIodS5fcmVuZGVyU2VydmljZS5vbkRpbWVuc2lvbnNDaGFuZ2UoKGZ1bmN0aW9uKGUpe3JldHVybiB1Ll9yZW5kZXJEaW1lbnNpb25zPWV9KSkpLHNldFRpbWVvdXQoKGZ1bmN0aW9uKCl7cmV0dXJuIHUuc3luY1Njcm9sbEFyZWEoKX0pLDApLHV9cmV0dXJuIG4odCxlKSx0LnByb3RvdHlwZS5vblRoZW1lQ2hhbmdlPWZ1bmN0aW9uKGUpe3RoaXMuX3ZpZXdwb3J0RWxlbWVudC5zdHlsZS5iYWNrZ3JvdW5kQ29sb3I9ZS5iYWNrZ3JvdW5kLmNzc30sdC5wcm90b3R5cGUuX3JlZnJlc2g9ZnVuY3Rpb24oZSl7dmFyIHQ9dGhpcztpZihlKXJldHVybiB0aGlzLl9pbm5lclJlZnJlc2goKSx2b2lkKG51bGwhPT10aGlzLl9yZWZyZXNoQW5pbWF0aW9uRnJhbWUmJmNhbmNlbEFuaW1hdGlvbkZyYW1lKHRoaXMuX3JlZnJlc2hBbmltYXRpb25GcmFtZSkpO251bGw9PT10aGlzLl9yZWZyZXNoQW5pbWF0aW9uRnJhbWUmJih0aGlzLl9yZWZyZXNoQW5pbWF0aW9uRnJhbWU9cmVxdWVzdEFuaW1hdGlvbkZyYW1lKChmdW5jdGlvbigpe3JldHVybiB0Ll9pbm5lclJlZnJlc2goKX0pKSl9LHQucHJvdG90eXBlLl9pbm5lclJlZnJlc2g9ZnVuY3Rpb24oKXtpZih0aGlzLl9jaGFyU2l6ZVNlcnZpY2UuaGVpZ2h0PjApe3RoaXMuX2N1cnJlbnRSb3dIZWlnaHQ9dGhpcy5fcmVuZGVyU2VydmljZS5kaW1lbnNpb25zLnNjYWxlZENlbGxIZWlnaHQvd2luZG93LmRldmljZVBpeGVsUmF0aW8sdGhpcy5fY3VycmVudFNjYWxlZENlbGxIZWlnaHQ9dGhpcy5fcmVuZGVyU2VydmljZS5kaW1lbnNpb25zLnNjYWxlZENlbGxIZWlnaHQsdGhpcy5fbGFzdFJlY29yZGVkVmlld3BvcnRIZWlnaHQ9dGhpcy5fdmlld3BvcnRFbGVtZW50Lm9mZnNldEhlaWdodDt2YXIgZT1NYXRoLnJvdW5kKHRoaXMuX2N1cnJlbnRSb3dIZWlnaHQqdGhpcy5fbGFzdFJlY29yZGVkQnVmZmVyTGVuZ3RoKSsodGhpcy5fbGFzdFJlY29yZGVkVmlld3BvcnRIZWlnaHQtdGhpcy5fcmVuZGVyU2VydmljZS5kaW1lbnNpb25zLmNhbnZhc0hlaWdodCk7dGhpcy5fbGFzdFJlY29yZGVkQnVmZmVySGVpZ2h0IT09ZSYmKHRoaXMuX2xhc3RSZWNvcmRlZEJ1ZmZlckhlaWdodD1lLHRoaXMuX3Njcm9sbEFyZWEuc3R5bGUuaGVpZ2h0PXRoaXMuX2xhc3RSZWNvcmRlZEJ1ZmZlckhlaWdodCsicHgiKX12YXIgdD10aGlzLl9idWZmZXJTZXJ2aWNlLmJ1ZmZlci55ZGlzcCp0aGlzLl9jdXJyZW50Um93SGVpZ2h0O3RoaXMuX3ZpZXdwb3J0RWxlbWVudC5zY3JvbGxUb3AhPT10JiYodGhpcy5faWdub3JlTmV4dFNjcm9sbEV2ZW50PSEwLHRoaXMuX3ZpZXdwb3J0RWxlbWVudC5zY3JvbGxUb3A9dCksMD09PXRoaXMuX29wdGlvbnNTZXJ2aWNlLm9wdGlvbnMuc2Nyb2xsYmFjaz90aGlzLnNjcm9sbEJhcldpZHRoPTA6dGhpcy5zY3JvbGxCYXJXaWR0aD10aGlzLl92aWV3cG9ydEVsZW1lbnQub2Zmc2V0V2lkdGgtdGhpcy5fc2Nyb2xsQXJlYS5vZmZzZXRXaWR0aHx8MTUsdGhpcy5fbGFzdEhhZFNjcm9sbEJhcj10aGlzLnNjcm9sbEJhcldpZHRoPjA7dmFyIHI9d2luZG93LmdldENvbXB1dGVkU3R5bGUodGhpcy5fZWxlbWVudCksaT1wYXJzZUludChyLnBhZGRpbmdMZWZ0KStwYXJzZUludChyLnBhZGRpbmdSaWdodCk7dGhpcy5fdmlld3BvcnRFbGVtZW50LnN0eWxlLndpZHRoPSh0aGlzLl9yZW5kZXJTZXJ2aWNlLmRpbWVuc2lvbnMuYWN0dWFsQ2VsbFdpZHRoKnRoaXMuX2J1ZmZlclNlcnZpY2UuY29scyt0aGlzLnNjcm9sbEJhcldpZHRoKyh0aGlzLl9sYXN0SGFkU2Nyb2xsQmFyP2k6MCkpLnRvU3RyaW5nKCkrInB4Iix0aGlzLl9yZWZyZXNoQW5pbWF0aW9uRnJhbWU9bnVsbH0sdC5wcm90b3R5cGUuc3luY1Njcm9sbEFyZWE9ZnVuY3Rpb24oZSl7aWYodm9pZCAwPT09ZSYmKGU9ITEpLHRoaXMuX2xhc3RSZWNvcmRlZEJ1ZmZlckxlbmd0aCE9PXRoaXMuX2J1ZmZlclNlcnZpY2UuYnVmZmVyLmxpbmVzLmxlbmd0aClyZXR1cm4gdGhpcy5fbGFzdFJlY29yZGVkQnVmZmVyTGVuZ3RoPXRoaXMuX2J1ZmZlclNlcnZpY2UuYnVmZmVyLmxpbmVzLmxlbmd0aCx2b2lkIHRoaXMuX3JlZnJlc2goZSk7dGhpcy5fbGFzdFJlY29yZGVkVmlld3BvcnRIZWlnaHQ9PT10aGlzLl9yZW5kZXJTZXJ2aWNlLmRpbWVuc2lvbnMuY2FudmFzSGVpZ2h0JiZ0aGlzLl9sYXN0U2Nyb2xsVG9wPT09dGhpcy5fYWN0aXZlQnVmZmVyLnlkaXNwKnRoaXMuX2N1cnJlbnRSb3dIZWlnaHQmJnRoaXMuX3JlbmRlckRpbWVuc2lvbnMuc2NhbGVkQ2VsbEhlaWdodD09PXRoaXMuX2N1cnJlbnRTY2FsZWRDZWxsSGVpZ2h0P3RoaXMuX2xhc3RIYWRTY3JvbGxCYXIhPT10aGlzLl9vcHRpb25zU2VydmljZS5vcHRpb25zLnNjcm9sbGJhY2s+MCYmdGhpcy5fcmVmcmVzaChlKTp0aGlzLl9yZWZyZXNoKGUpfSx0LnByb3RvdHlwZS5fb25TY3JvbGw9ZnVuY3Rpb24oZSl7aWYodGhpcy5fbGFzdFNjcm9sbFRvcD10aGlzLl92aWV3cG9ydEVsZW1lbnQuc2Nyb2xsVG9wLHRoaXMuX3ZpZXdwb3J0RWxlbWVudC5vZmZzZXRQYXJlbnQpe2lmKHRoaXMuX2lnbm9yZU5leHRTY3JvbGxFdmVudClyZXR1cm4gdGhpcy5faWdub3JlTmV4dFNjcm9sbEV2ZW50PSExLHZvaWQgdGhpcy5fc2Nyb2xsTGluZXMoMCk7dmFyIHQ9TWF0aC5yb3VuZCh0aGlzLl9sYXN0U2Nyb2xsVG9wL3RoaXMuX2N1cnJlbnRSb3dIZWlnaHQpLXRoaXMuX2J1ZmZlclNlcnZpY2UuYnVmZmVyLnlkaXNwO3RoaXMuX3Njcm9sbExpbmVzKHQpfX0sdC5wcm90b3R5cGUuX2J1YmJsZVNjcm9sbD1mdW5jdGlvbihlLHQpe3ZhciByPXRoaXMuX3ZpZXdwb3J0RWxlbWVudC5zY3JvbGxUb3ArdGhpcy5fbGFzdFJlY29yZGVkVmlld3BvcnRIZWlnaHQ7cmV0dXJuISh0PDAmJjAhPT10aGlzLl92aWV3cG9ydEVsZW1lbnQuc2Nyb2xsVG9wfHx0PjAmJnI8dGhpcy5fbGFzdFJlY29yZGVkQnVmZmVySGVpZ2h0KXx8KGUuY2FuY2VsYWJsZSYmZS5wcmV2ZW50RGVmYXVsdCgpLCExKX0sdC5wcm90b3R5cGUub25XaGVlbD1mdW5jdGlvbihlKXt2YXIgdD10aGlzLl9nZXRQaXhlbHNTY3JvbGxlZChlKTtyZXR1cm4gMCE9PXQmJih0aGlzLl92aWV3cG9ydEVsZW1lbnQuc2Nyb2xsVG9wKz10LHRoaXMuX2J1YmJsZVNjcm9sbChlLHQpKX0sdC5wcm90b3R5cGUuX2dldFBpeGVsc1Njcm9sbGVkPWZ1bmN0aW9uKGUpe2lmKDA9PT1lLmRlbHRhWXx8ZS5zaGlmdEtleSlyZXR1cm4gMDt2YXIgdD10aGlzLl9hcHBseVNjcm9sbE1vZGlmaWVyKGUuZGVsdGFZLGUpO3JldHVybiBlLmRlbHRhTW9kZT09PVdoZWVsRXZlbnQuRE9NX0RFTFRBX0xJTkU/dCo9dGhpcy5fY3VycmVudFJvd0hlaWdodDplLmRlbHRhTW9kZT09PVdoZWVsRXZlbnQuRE9NX0RFTFRBX1BBR0UmJih0Kj10aGlzLl9jdXJyZW50Um93SGVpZ2h0KnRoaXMuX2J1ZmZlclNlcnZpY2Uucm93cyksdH0sdC5wcm90b3R5cGUuZ2V0TGluZXNTY3JvbGxlZD1mdW5jdGlvbihlKXtpZigwPT09ZS5kZWx0YVl8fGUuc2hpZnRLZXkpcmV0dXJuIDA7dmFyIHQ9dGhpcy5fYXBwbHlTY3JvbGxNb2RpZmllcihlLmRlbHRhWSxlKTtyZXR1cm4gZS5kZWx0YU1vZGU9PT1XaGVlbEV2ZW50LkRPTV9ERUxUQV9QSVhFTD8odC89dGhpcy5fY3VycmVudFJvd0hlaWdodCswLHRoaXMuX3doZWVsUGFydGlhbFNjcm9sbCs9dCx0PU1hdGguZmxvb3IoTWF0aC5hYnModGhpcy5fd2hlZWxQYXJ0aWFsU2Nyb2xsKSkqKHRoaXMuX3doZWVsUGFydGlhbFNjcm9sbD4wPzE6LTEpLHRoaXMuX3doZWVsUGFydGlhbFNjcm9sbCU9MSk6ZS5kZWx0YU1vZGU9PT1XaGVlbEV2ZW50LkRPTV9ERUxUQV9QQUdFJiYodCo9dGhpcy5fYnVmZmVyU2VydmljZS5yb3dzKSx0fSx0LnByb3RvdHlwZS5fYXBwbHlTY3JvbGxNb2RpZmllcj1mdW5jdGlvbihlLHQpe3ZhciByPXRoaXMuX29wdGlvbnNTZXJ2aWNlLm9wdGlvbnMuZmFzdFNjcm9sbE1vZGlmaWVyO3JldHVybiJhbHQiPT09ciYmdC5hbHRLZXl8fCJjdHJsIj09PXImJnQuY3RybEtleXx8InNoaWZ0Ij09PXImJnQuc2hpZnRLZXk/ZSp0aGlzLl9vcHRpb25zU2VydmljZS5vcHRpb25zLmZhc3RTY3JvbGxTZW5zaXRpdml0eSp0aGlzLl9vcHRpb25zU2VydmljZS5vcHRpb25zLnNjcm9sbFNlbnNpdGl2aXR5OmUqdGhpcy5fb3B0aW9uc1NlcnZpY2Uub3B0aW9ucy5zY3JvbGxTZW5zaXRpdml0eX0sdC5wcm90b3R5cGUub25Ub3VjaFN0YXJ0PWZ1bmN0aW9uKGUpe3RoaXMuX2xhc3RUb3VjaFk9ZS50b3VjaGVzWzBdLnBhZ2VZfSx0LnByb3RvdHlwZS5vblRvdWNoTW92ZT1mdW5jdGlvbihlKXt2YXIgdD10aGlzLl9sYXN0VG91Y2hZLWUudG91Y2hlc1swXS5wYWdlWTtyZXR1cm4gdGhpcy5fbGFzdFRvdWNoWT1lLnRvdWNoZXNbMF0ucGFnZVksMCE9PXQmJih0aGlzLl92aWV3cG9ydEVsZW1lbnQuc2Nyb2xsVG9wKz10LHRoaXMuX2J1YmJsZVNjcm9sbChlLHQpKX0sbyhbcyg0LHUuSUJ1ZmZlclNlcnZpY2UpLHMoNSx1LklPcHRpb25zU2VydmljZSkscyg2LGwuSUNoYXJTaXplU2VydmljZSkscyg3LGwuSVJlbmRlclNlcnZpY2UpXSx0KX0oYS5EaXNwb3NhYmxlKTt0LlZpZXdwb3J0PWh9LDI5NTA6ZnVuY3Rpb24oZSx0LHIpe3ZhciBpPXRoaXMmJnRoaXMuX19kZWNvcmF0ZXx8ZnVuY3Rpb24oZSx0LHIsaSl7dmFyIG4sbz1hcmd1bWVudHMubGVuZ3RoLHM9bzwzP3Q6bnVsbD09PWk/aT1PYmplY3QuZ2V0T3duUHJvcGVydHlEZXNjcmlwdG9yKHQscik6aTtpZigib2JqZWN0Ij09dHlwZW9mIFJlZmxlY3QmJiJmdW5jdGlvbiI9PXR5cGVvZiBSZWZsZWN0LmRlY29yYXRlKXM9UmVmbGVjdC5kZWNvcmF0ZShlLHQscixpKTtlbHNlIGZvcih2YXIgYT1lLmxlbmd0aC0xO2E+PTA7YS0tKShuPWVbYV0pJiYocz0obzwzP24ocyk6bz4zP24odCxyLHMpOm4odCxyKSl8fHMpO3JldHVybiBvPjMmJnMmJk9iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LHIscyksc30sbj10aGlzJiZ0aGlzLl9fcGFyYW18fGZ1bmN0aW9uKGUsdCl7cmV0dXJuIGZ1bmN0aW9uKHIsaSl7dChyLGksZSl9fTtPYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC5Db21wb3NpdGlvbkhlbHBlcj12b2lkIDA7dmFyIG89cig0NzI1KSxzPXIoMjU4NSksYT1mdW5jdGlvbigpe2Z1bmN0aW9uIGUoZSx0LHIsaSxuLG8pe3RoaXMuX3RleHRhcmVhPWUsdGhpcy5fY29tcG9zaXRpb25WaWV3PXQsdGhpcy5fYnVmZmVyU2VydmljZT1yLHRoaXMuX29wdGlvbnNTZXJ2aWNlPWksdGhpcy5fY29yZVNlcnZpY2U9bix0aGlzLl9yZW5kZXJTZXJ2aWNlPW8sdGhpcy5faXNDb21wb3Npbmc9ITEsdGhpcy5faXNTZW5kaW5nQ29tcG9zaXRpb249ITEsdGhpcy5fY29tcG9zaXRpb25Qb3NpdGlvbj17c3RhcnQ6MCxlbmQ6MH0sdGhpcy5fZGF0YUFscmVhZHlTZW50PSIifXJldHVybiBPYmplY3QuZGVmaW5lUHJvcGVydHkoZS5wcm90b3R5cGUsImlzQ29tcG9zaW5nIix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX2lzQ29tcG9zaW5nfSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLGUucHJvdG90eXBlLmNvbXBvc2l0aW9uc3RhcnQ9ZnVuY3Rpb24oKXt0aGlzLl9pc0NvbXBvc2luZz0hMCx0aGlzLl9jb21wb3NpdGlvblBvc2l0aW9uLnN0YXJ0PXRoaXMuX3RleHRhcmVhLnZhbHVlLmxlbmd0aCx0aGlzLl9jb21wb3NpdGlvblZpZXcudGV4dENvbnRlbnQ9IiIsdGhpcy5fZGF0YUFscmVhZHlTZW50PSIiLHRoaXMuX2NvbXBvc2l0aW9uVmlldy5jbGFzc0xpc3QuYWRkKCJhY3RpdmUiKX0sZS5wcm90b3R5cGUuY29tcG9zaXRpb251cGRhdGU9ZnVuY3Rpb24oZSl7dmFyIHQ9dGhpczt0aGlzLl9jb21wb3NpdGlvblZpZXcudGV4dENvbnRlbnQ9ZS5kYXRhLHRoaXMudXBkYXRlQ29tcG9zaXRpb25FbGVtZW50cygpLHNldFRpbWVvdXQoKGZ1bmN0aW9uKCl7dC5fY29tcG9zaXRpb25Qb3NpdGlvbi5lbmQ9dC5fdGV4dGFyZWEudmFsdWUubGVuZ3RofSksMCl9LGUucHJvdG90eXBlLmNvbXBvc2l0aW9uZW5kPWZ1bmN0aW9uKCl7dGhpcy5fZmluYWxpemVDb21wb3NpdGlvbighMCl9LGUucHJvdG90eXBlLmtleWRvd249ZnVuY3Rpb24oZSl7aWYodGhpcy5faXNDb21wb3Npbmd8fHRoaXMuX2lzU2VuZGluZ0NvbXBvc2l0aW9uKXtpZigyMjk9PT1lLmtleUNvZGUpcmV0dXJuITE7aWYoMTY9PT1lLmtleUNvZGV8fDE3PT09ZS5rZXlDb2RlfHwxOD09PWUua2V5Q29kZSlyZXR1cm4hMTt0aGlzLl9maW5hbGl6ZUNvbXBvc2l0aW9uKCExKX1yZXR1cm4gMjI5IT09ZS5rZXlDb2RlfHwodGhpcy5faGFuZGxlQW55VGV4dGFyZWFDaGFuZ2VzKCksITEpfSxlLnByb3RvdHlwZS5fZmluYWxpemVDb21wb3NpdGlvbj1mdW5jdGlvbihlKXt2YXIgdD10aGlzO2lmKHRoaXMuX2NvbXBvc2l0aW9uVmlldy5jbGFzc0xpc3QucmVtb3ZlKCJhY3RpdmUiKSx0aGlzLl9pc0NvbXBvc2luZz0hMSxlKXt2YXIgcj17c3RhcnQ6dGhpcy5fY29tcG9zaXRpb25Qb3NpdGlvbi5zdGFydCxlbmQ6dGhpcy5fY29tcG9zaXRpb25Qb3NpdGlvbi5lbmR9O3RoaXMuX2lzU2VuZGluZ0NvbXBvc2l0aW9uPSEwLHNldFRpbWVvdXQoKGZ1bmN0aW9uKCl7dmFyIGU7dC5faXNTZW5kaW5nQ29tcG9zaXRpb24mJih0Ll9pc1NlbmRpbmdDb21wb3NpdGlvbj0hMSxyLnN0YXJ0Kz10Ll9kYXRhQWxyZWFkeVNlbnQubGVuZ3RoLChlPXQuX2lzQ29tcG9zaW5nP3QuX3RleHRhcmVhLnZhbHVlLnN1YnN0cmluZyhyLnN0YXJ0LHIuZW5kKTp0Ll90ZXh0YXJlYS52YWx1ZS5zdWJzdHJpbmcoci5zdGFydCkpLmxlbmd0aD4wJiZ0Ll9jb3JlU2VydmljZS50cmlnZ2VyRGF0YUV2ZW50KGUsITApKX0pLDApfWVsc2V7dGhpcy5faXNTZW5kaW5nQ29tcG9zaXRpb249ITE7dmFyIGk9dGhpcy5fdGV4dGFyZWEudmFsdWUuc3Vic3RyaW5nKHRoaXMuX2NvbXBvc2l0aW9uUG9zaXRpb24uc3RhcnQsdGhpcy5fY29tcG9zaXRpb25Qb3NpdGlvbi5lbmQpO3RoaXMuX2NvcmVTZXJ2aWNlLnRyaWdnZXJEYXRhRXZlbnQoaSwhMCl9fSxlLnByb3RvdHlwZS5faGFuZGxlQW55VGV4dGFyZWFDaGFuZ2VzPWZ1bmN0aW9uKCl7dmFyIGU9dGhpcyx0PXRoaXMuX3RleHRhcmVhLnZhbHVlO3NldFRpbWVvdXQoKGZ1bmN0aW9uKCl7aWYoIWUuX2lzQ29tcG9zaW5nKXt2YXIgcj1lLl90ZXh0YXJlYS52YWx1ZS5yZXBsYWNlKHQsIiIpO3IubGVuZ3RoPjAmJihlLl9kYXRhQWxyZWFkeVNlbnQ9cixlLl9jb3JlU2VydmljZS50cmlnZ2VyRGF0YUV2ZW50KHIsITApKX19KSwwKX0sZS5wcm90b3R5cGUudXBkYXRlQ29tcG9zaXRpb25FbGVtZW50cz1mdW5jdGlvbihlKXt2YXIgdD10aGlzO2lmKHRoaXMuX2lzQ29tcG9zaW5nKXtpZih0aGlzLl9idWZmZXJTZXJ2aWNlLmJ1ZmZlci5pc0N1cnNvckluVmlld3BvcnQpe3ZhciByPU1hdGgubWluKHRoaXMuX2J1ZmZlclNlcnZpY2UuYnVmZmVyLngsdGhpcy5fYnVmZmVyU2VydmljZS5jb2xzLTEpLGk9dGhpcy5fcmVuZGVyU2VydmljZS5kaW1lbnNpb25zLmFjdHVhbENlbGxIZWlnaHQsbj10aGlzLl9idWZmZXJTZXJ2aWNlLmJ1ZmZlci55KnRoaXMuX3JlbmRlclNlcnZpY2UuZGltZW5zaW9ucy5hY3R1YWxDZWxsSGVpZ2h0LG89cip0aGlzLl9yZW5kZXJTZXJ2aWNlLmRpbWVuc2lvbnMuYWN0dWFsQ2VsbFdpZHRoO3RoaXMuX2NvbXBvc2l0aW9uVmlldy5zdHlsZS5sZWZ0PW8rInB4Iix0aGlzLl9jb21wb3NpdGlvblZpZXcuc3R5bGUudG9wPW4rInB4Iix0aGlzLl9jb21wb3NpdGlvblZpZXcuc3R5bGUuaGVpZ2h0PWkrInB4Iix0aGlzLl9jb21wb3NpdGlvblZpZXcuc3R5bGUubGluZUhlaWdodD1pKyJweCIsdGhpcy5fY29tcG9zaXRpb25WaWV3LnN0eWxlLmZvbnRGYW1pbHk9dGhpcy5fb3B0aW9uc1NlcnZpY2Uub3B0aW9ucy5mb250RmFtaWx5LHRoaXMuX2NvbXBvc2l0aW9uVmlldy5zdHlsZS5mb250U2l6ZT10aGlzLl9vcHRpb25zU2VydmljZS5vcHRpb25zLmZvbnRTaXplKyJweCI7dmFyIHM9dGhpcy5fY29tcG9zaXRpb25WaWV3LmdldEJvdW5kaW5nQ2xpZW50UmVjdCgpO3RoaXMuX3RleHRhcmVhLnN0eWxlLmxlZnQ9bysicHgiLHRoaXMuX3RleHRhcmVhLnN0eWxlLnRvcD1uKyJweCIsdGhpcy5fdGV4dGFyZWEuc3R5bGUud2lkdGg9TWF0aC5tYXgocy53aWR0aCwxKSsicHgiLHRoaXMuX3RleHRhcmVhLnN0eWxlLmhlaWdodD1NYXRoLm1heChzLmhlaWdodCwxKSsicHgiLHRoaXMuX3RleHRhcmVhLnN0eWxlLmxpbmVIZWlnaHQ9cy5oZWlnaHQrInB4In1lfHxzZXRUaW1lb3V0KChmdW5jdGlvbigpe3JldHVybiB0LnVwZGF0ZUNvbXBvc2l0aW9uRWxlbWVudHMoITApfSksMCl9fSxpKFtuKDIscy5JQnVmZmVyU2VydmljZSksbigzLHMuSU9wdGlvbnNTZXJ2aWNlKSxuKDQscy5JQ29yZVNlcnZpY2UpLG4oNSxvLklSZW5kZXJTZXJ2aWNlKV0sZSl9KCk7dC5Db21wb3NpdGlvbkhlbHBlcj1hfSw5ODA2OihlLHQpPT57ZnVuY3Rpb24gcihlLHQpe3ZhciByPXQuZ2V0Qm91bmRpbmdDbGllbnRSZWN0KCk7cmV0dXJuW2UuY2xpZW50WC1yLmxlZnQsZS5jbGllbnRZLXIudG9wXX1PYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC5nZXRSYXdCeXRlQ29vcmRzPXQuZ2V0Q29vcmRzPXQuZ2V0Q29vcmRzUmVsYXRpdmVUb0VsZW1lbnQ9dm9pZCAwLHQuZ2V0Q29vcmRzUmVsYXRpdmVUb0VsZW1lbnQ9cix0LmdldENvb3Jkcz1mdW5jdGlvbihlLHQsaSxuLG8scyxhLGMpe2lmKG8pe3ZhciBsPXIoZSx0KTtpZihsKXJldHVybiBsWzBdPU1hdGguY2VpbCgobFswXSsoYz9zLzI6MCkpL3MpLGxbMV09TWF0aC5jZWlsKGxbMV0vYSksbFswXT1NYXRoLm1pbihNYXRoLm1heChsWzBdLDEpLGkrKGM/MTowKSksbFsxXT1NYXRoLm1pbihNYXRoLm1heChsWzFdLDEpLG4pLGx9fSx0LmdldFJhd0J5dGVDb29yZHM9ZnVuY3Rpb24oZSl7aWYoZSlyZXR1cm57eDplWzBdKzMyLHk6ZVsxXSszMn19fSw5NTA0OihlLHQscik9PntPYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC5tb3ZlVG9DZWxsU2VxdWVuY2U9dm9pZCAwO3ZhciBpPXIoMjU4NCk7ZnVuY3Rpb24gbihlLHQscixpKXt2YXIgbj1lLW8ocixlKSxhPXQtbyhyLHQpLHU9TWF0aC5hYnMobi1hKS1mdW5jdGlvbihlLHQscil7Zm9yKHZhciBpPTAsbj1lLW8ocixlKSxhPXQtbyhyLHQpLGM9MDtjPE1hdGguYWJzKG4tYSk7YysrKXt2YXIgbD0iQSI9PT1zKGUsdCk/LTE6MSx1PXIuYnVmZmVyLmxpbmVzLmdldChuK2wqYyk7KG51bGw9PXU/dm9pZCAwOnUuaXNXcmFwcGVkKSYmaSsrfXJldHVybiBpfShlLHQscik7cmV0dXJuIGwodSxjKHMoZSx0KSxpKSl9ZnVuY3Rpb24gbyhlLHQpe2Zvcih2YXIgcj0wLGk9ZS5idWZmZXIubGluZXMuZ2V0KHQpLG49bnVsbD09aT92b2lkIDA6aS5pc1dyYXBwZWQ7biYmdD49MCYmdDxlLnJvd3M7KXIrKyxuPW51bGw9PShpPWUuYnVmZmVyLmxpbmVzLmdldCgtLXQpKT92b2lkIDA6aS5pc1dyYXBwZWQ7cmV0dXJuIHJ9ZnVuY3Rpb24gcyhlLHQpe3JldHVybiBlPnQ/IkEiOiJCIn1mdW5jdGlvbiBhKGUsdCxyLGksbixvKXtmb3IodmFyIHM9ZSxhPXQsYz0iIjtzIT09cnx8YSE9PWk7KXMrPW4/MTotMSxuJiZzPm8uY29scy0xPyhjKz1vLmJ1ZmZlci50cmFuc2xhdGVCdWZmZXJMaW5lVG9TdHJpbmcoYSwhMSxlLHMpLHM9MCxlPTAsYSsrKTohbiYmczwwJiYoYys9by5idWZmZXIudHJhbnNsYXRlQnVmZmVyTGluZVRvU3RyaW5nKGEsITEsMCxlKzEpLGU9cz1vLmNvbHMtMSxhLS0pO3JldHVybiBjK28uYnVmZmVyLnRyYW5zbGF0ZUJ1ZmZlckxpbmVUb1N0cmluZyhhLCExLGUscyl9ZnVuY3Rpb24gYyhlLHQpe3ZhciByPXQ/Ik8iOiJbIjtyZXR1cm4gaS5DMC5FU0MrcitlfWZ1bmN0aW9uIGwoZSx0KXtlPU1hdGguZmxvb3IoZSk7Zm9yKHZhciByPSIiLGk9MDtpPGU7aSsrKXIrPXQ7cmV0dXJuIHJ9dC5tb3ZlVG9DZWxsU2VxdWVuY2U9ZnVuY3Rpb24oZSx0LHIsaSl7dmFyIHMsdT1yLmJ1ZmZlci54LGg9ci5idWZmZXIueTtpZighci5idWZmZXIuaGFzU2Nyb2xsYmFjaylyZXR1cm4gZnVuY3Rpb24oZSx0LHIsaSxzLHUpe3JldHVybiAwPT09bih0LGkscyx1KS5sZW5ndGg/IiI6bChhKGUsdCxlLHQtbyhzLHQpLCExLHMpLmxlbmd0aCxjKCJEIix1KSl9KHUsaCwwLHQscixpKStuKGgsdCxyLGkpK2Z1bmN0aW9uKGUsdCxyLGkscyx1KXt2YXIgaDtoPW4odCxpLHMsdSkubGVuZ3RoPjA/aS1vKHMsaSk6dDt2YXIgZj1pLF89ZnVuY3Rpb24oZSx0LHIsaSxzLGEpe3ZhciBjO3JldHVybiBjPW4ocixpLHMsYSkubGVuZ3RoPjA/aS1vKHMsaSk6dCxlPHImJmM8PWl8fGU+PXImJmM8aT8iQyI6IkQifShlLHQscixpLHMsdSk7cmV0dXJuIGwoYShlLGgscixmLCJDIj09PV8scykubGVuZ3RoLGMoXyx1KSl9KHUsaCxlLHQscixpKTtpZihoPT09dClyZXR1cm4gcz11PmU/IkQiOiJDIixsKE1hdGguYWJzKHUtZSksYyhzLGkpKTtzPWg+dD8iRCI6IkMiO3ZhciBmPU1hdGguYWJzKGgtdCk7cmV0dXJuIGwoZnVuY3Rpb24oZSx0KXtyZXR1cm4gdC5jb2xzLWV9KGg+dD9lOnUscikrKGYtMSkqci5jb2xzKzErKChoPnQ/dTplKS0xKSxjKHMsaSkpfX0sMTU0NjooZSx0LHIpPT57T2JqZWN0LmRlZmluZVByb3BlcnR5KHQsIl9fZXNNb2R1bGUiLHt2YWx1ZTohMH0pLHQuQmFzZVJlbmRlckxheWVyPXZvaWQgMDt2YXIgaT1yKDY0Myksbj1yKDg4MDMpLG89cigxNDIwKSxzPXIoMzczNCksYT1yKDE3NTIpLGM9cig0Nzc0KSxsPXIoOTYzMSksdT1yKDg5NzgpLGg9ZnVuY3Rpb24oKXtmdW5jdGlvbiBlKGUsdCxyLGksbixvLHMsYSl7dGhpcy5fY29udGFpbmVyPWUsdGhpcy5fYWxwaGE9aSx0aGlzLl9jb2xvcnM9bix0aGlzLl9yZW5kZXJlcklkPW8sdGhpcy5fYnVmZmVyU2VydmljZT1zLHRoaXMuX29wdGlvbnNTZXJ2aWNlPWEsdGhpcy5fc2NhbGVkQ2hhcldpZHRoPTAsdGhpcy5fc2NhbGVkQ2hhckhlaWdodD0wLHRoaXMuX3NjYWxlZENlbGxXaWR0aD0wLHRoaXMuX3NjYWxlZENlbGxIZWlnaHQ9MCx0aGlzLl9zY2FsZWRDaGFyTGVmdD0wLHRoaXMuX3NjYWxlZENoYXJUb3A9MCx0aGlzLl9jdXJyZW50R2x5cGhJZGVudGlmaWVyPXtjaGFyczoiIixjb2RlOjAsYmc6MCxmZzowLGJvbGQ6ITEsZGltOiExLGl0YWxpYzohMX0sdGhpcy5fY2FudmFzPWRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoImNhbnZhcyIpLHRoaXMuX2NhbnZhcy5jbGFzc0xpc3QuYWRkKCJ4dGVybS0iK3QrIi1sYXllciIpLHRoaXMuX2NhbnZhcy5zdHlsZS56SW5kZXg9ci50b1N0cmluZygpLHRoaXMuX2luaXRDYW52YXMoKSx0aGlzLl9jb250YWluZXIuYXBwZW5kQ2hpbGQodGhpcy5fY2FudmFzKX1yZXR1cm4gZS5wcm90b3R5cGUuZGlzcG9zZT1mdW5jdGlvbigpe3ZhciBlOygwLGwucmVtb3ZlRWxlbWVudEZyb21QYXJlbnQpKHRoaXMuX2NhbnZhcyksbnVsbD09PShlPXRoaXMuX2NoYXJBdGxhcyl8fHZvaWQgMD09PWV8fGUuZGlzcG9zZSgpfSxlLnByb3RvdHlwZS5faW5pdENhbnZhcz1mdW5jdGlvbigpe3RoaXMuX2N0eD0oMCxhLnRocm93SWZGYWxzeSkodGhpcy5fY2FudmFzLmdldENvbnRleHQoIjJkIix7YWxwaGE6dGhpcy5fYWxwaGF9KSksdGhpcy5fYWxwaGF8fHRoaXMuX2NsZWFyQWxsKCl9LGUucHJvdG90eXBlLm9uT3B0aW9uc0NoYW5nZWQ9ZnVuY3Rpb24oKXt9LGUucHJvdG90eXBlLm9uQmx1cj1mdW5jdGlvbigpe30sZS5wcm90b3R5cGUub25Gb2N1cz1mdW5jdGlvbigpe30sZS5wcm90b3R5cGUub25DdXJzb3JNb3ZlPWZ1bmN0aW9uKCl7fSxlLnByb3RvdHlwZS5vbkdyaWRDaGFuZ2VkPWZ1bmN0aW9uKGUsdCl7fSxlLnByb3RvdHlwZS5vblNlbGVjdGlvbkNoYW5nZWQ9ZnVuY3Rpb24oZSx0LHIpe3ZvaWQgMD09PXImJihyPSExKX0sZS5wcm90b3R5cGUuc2V0Q29sb3JzPWZ1bmN0aW9uKGUpe3RoaXMuX3JlZnJlc2hDaGFyQXRsYXMoZSl9LGUucHJvdG90eXBlLl9zZXRUcmFuc3BhcmVuY3k9ZnVuY3Rpb24oZSl7aWYoZSE9PXRoaXMuX2FscGhhKXt2YXIgdD10aGlzLl9jYW52YXM7dGhpcy5fYWxwaGE9ZSx0aGlzLl9jYW52YXM9dGhpcy5fY2FudmFzLmNsb25lTm9kZSgpLHRoaXMuX2luaXRDYW52YXMoKSx0aGlzLl9jb250YWluZXIucmVwbGFjZUNoaWxkKHRoaXMuX2NhbnZhcyx0KSx0aGlzLl9yZWZyZXNoQ2hhckF0bGFzKHRoaXMuX2NvbG9ycyksdGhpcy5vbkdyaWRDaGFuZ2VkKDAsdGhpcy5fYnVmZmVyU2VydmljZS5yb3dzLTEpfX0sZS5wcm90b3R5cGUuX3JlZnJlc2hDaGFyQXRsYXM9ZnVuY3Rpb24oZSl7dGhpcy5fc2NhbGVkQ2hhcldpZHRoPD0wJiZ0aGlzLl9zY2FsZWRDaGFySGVpZ2h0PD0wfHwodGhpcy5fY2hhckF0bGFzPSgwLG8uYWNxdWlyZUNoYXJBdGxhcykodGhpcy5fb3B0aW9uc1NlcnZpY2Uub3B0aW9ucyx0aGlzLl9yZW5kZXJlcklkLGUsdGhpcy5fc2NhbGVkQ2hhcldpZHRoLHRoaXMuX3NjYWxlZENoYXJIZWlnaHQpLHRoaXMuX2NoYXJBdGxhcy53YXJtVXAoKSl9LGUucHJvdG90eXBlLnJlc2l6ZT1mdW5jdGlvbihlKXt0aGlzLl9zY2FsZWRDZWxsV2lkdGg9ZS5zY2FsZWRDZWxsV2lkdGgsdGhpcy5fc2NhbGVkQ2VsbEhlaWdodD1lLnNjYWxlZENlbGxIZWlnaHQsdGhpcy5fc2NhbGVkQ2hhcldpZHRoPWUuc2NhbGVkQ2hhcldpZHRoLHRoaXMuX3NjYWxlZENoYXJIZWlnaHQ9ZS5zY2FsZWRDaGFySGVpZ2h0LHRoaXMuX3NjYWxlZENoYXJMZWZ0PWUuc2NhbGVkQ2hhckxlZnQsdGhpcy5fc2NhbGVkQ2hhclRvcD1lLnNjYWxlZENoYXJUb3AsdGhpcy5fY2FudmFzLndpZHRoPWUuc2NhbGVkQ2FudmFzV2lkdGgsdGhpcy5fY2FudmFzLmhlaWdodD1lLnNjYWxlZENhbnZhc0hlaWdodCx0aGlzLl9jYW52YXMuc3R5bGUud2lkdGg9ZS5jYW52YXNXaWR0aCsicHgiLHRoaXMuX2NhbnZhcy5zdHlsZS5oZWlnaHQ9ZS5jYW52YXNIZWlnaHQrInB4Iix0aGlzLl9hbHBoYXx8dGhpcy5fY2xlYXJBbGwoKSx0aGlzLl9yZWZyZXNoQ2hhckF0bGFzKHRoaXMuX2NvbG9ycyl9LGUucHJvdG90eXBlLmNsZWFyVGV4dHVyZUF0bGFzPWZ1bmN0aW9uKCl7dmFyIGU7bnVsbD09PShlPXRoaXMuX2NoYXJBdGxhcyl8fHZvaWQgMD09PWV8fGUuY2xlYXIoKX0sZS5wcm90b3R5cGUuX2ZpbGxDZWxscz1mdW5jdGlvbihlLHQscixpKXt0aGlzLl9jdHguZmlsbFJlY3QoZSp0aGlzLl9zY2FsZWRDZWxsV2lkdGgsdCp0aGlzLl9zY2FsZWRDZWxsSGVpZ2h0LHIqdGhpcy5fc2NhbGVkQ2VsbFdpZHRoLGkqdGhpcy5fc2NhbGVkQ2VsbEhlaWdodCl9LGUucHJvdG90eXBlLl9maWxsTWlkZGxlTGluZUF0Q2VsbHM9ZnVuY3Rpb24oZSx0LHIpe3ZvaWQgMD09PXImJihyPTEpO3ZhciBpPU1hdGguY2VpbCguNSp0aGlzLl9zY2FsZWRDZWxsSGVpZ2h0KTt0aGlzLl9jdHguZmlsbFJlY3QoZSp0aGlzLl9zY2FsZWRDZWxsV2lkdGgsKHQrMSkqdGhpcy5fc2NhbGVkQ2VsbEhlaWdodC1pLXdpbmRvdy5kZXZpY2VQaXhlbFJhdGlvLHIqdGhpcy5fc2NhbGVkQ2VsbFdpZHRoLHdpbmRvdy5kZXZpY2VQaXhlbFJhdGlvKX0sZS5wcm90b3R5cGUuX2ZpbGxCb3R0b21MaW5lQXRDZWxscz1mdW5jdGlvbihlLHQscil7dm9pZCAwPT09ciYmKHI9MSksdGhpcy5fY3R4LmZpbGxSZWN0KGUqdGhpcy5fc2NhbGVkQ2VsbFdpZHRoLCh0KzEpKnRoaXMuX3NjYWxlZENlbGxIZWlnaHQtd2luZG93LmRldmljZVBpeGVsUmF0aW8tMSxyKnRoaXMuX3NjYWxlZENlbGxXaWR0aCx3aW5kb3cuZGV2aWNlUGl4ZWxSYXRpbyl9LGUucHJvdG90eXBlLl9maWxsTGVmdExpbmVBdENlbGw9ZnVuY3Rpb24oZSx0LHIpe3RoaXMuX2N0eC5maWxsUmVjdChlKnRoaXMuX3NjYWxlZENlbGxXaWR0aCx0KnRoaXMuX3NjYWxlZENlbGxIZWlnaHQsd2luZG93LmRldmljZVBpeGVsUmF0aW8qcix0aGlzLl9zY2FsZWRDZWxsSGVpZ2h0KX0sZS5wcm90b3R5cGUuX3N0cm9rZVJlY3RBdENlbGw9ZnVuY3Rpb24oZSx0LHIsaSl7dGhpcy5fY3R4LmxpbmVXaWR0aD13aW5kb3cuZGV2aWNlUGl4ZWxSYXRpbyx0aGlzLl9jdHguc3Ryb2tlUmVjdChlKnRoaXMuX3NjYWxlZENlbGxXaWR0aCt3aW5kb3cuZGV2aWNlUGl4ZWxSYXRpby8yLHQqdGhpcy5fc2NhbGVkQ2VsbEhlaWdodCt3aW5kb3cuZGV2aWNlUGl4ZWxSYXRpby8yLHIqdGhpcy5fc2NhbGVkQ2VsbFdpZHRoLXdpbmRvdy5kZXZpY2VQaXhlbFJhdGlvLGkqdGhpcy5fc2NhbGVkQ2VsbEhlaWdodC13aW5kb3cuZGV2aWNlUGl4ZWxSYXRpbyl9LGUucHJvdG90eXBlLl9jbGVhckFsbD1mdW5jdGlvbigpe3RoaXMuX2FscGhhP3RoaXMuX2N0eC5jbGVhclJlY3QoMCwwLHRoaXMuX2NhbnZhcy53aWR0aCx0aGlzLl9jYW52YXMuaGVpZ2h0KToodGhpcy5fY3R4LmZpbGxTdHlsZT10aGlzLl9jb2xvcnMuYmFja2dyb3VuZC5jc3MsdGhpcy5fY3R4LmZpbGxSZWN0KDAsMCx0aGlzLl9jYW52YXMud2lkdGgsdGhpcy5fY2FudmFzLmhlaWdodCkpfSxlLnByb3RvdHlwZS5fY2xlYXJDZWxscz1mdW5jdGlvbihlLHQscixpKXt0aGlzLl9hbHBoYT90aGlzLl9jdHguY2xlYXJSZWN0KGUqdGhpcy5fc2NhbGVkQ2VsbFdpZHRoLHQqdGhpcy5fc2NhbGVkQ2VsbEhlaWdodCxyKnRoaXMuX3NjYWxlZENlbGxXaWR0aCxpKnRoaXMuX3NjYWxlZENlbGxIZWlnaHQpOih0aGlzLl9jdHguZmlsbFN0eWxlPXRoaXMuX2NvbG9ycy5iYWNrZ3JvdW5kLmNzcyx0aGlzLl9jdHguZmlsbFJlY3QoZSp0aGlzLl9zY2FsZWRDZWxsV2lkdGgsdCp0aGlzLl9zY2FsZWRDZWxsSGVpZ2h0LHIqdGhpcy5fc2NhbGVkQ2VsbFdpZHRoLGkqdGhpcy5fc2NhbGVkQ2VsbEhlaWdodCkpfSxlLnByb3RvdHlwZS5fZmlsbENoYXJUcnVlQ29sb3I9ZnVuY3Rpb24oZSx0LHIpe3RoaXMuX2N0eC5mb250PXRoaXMuX2dldEZvbnQoITEsITEpLHRoaXMuX2N0eC50ZXh0QmFzZWxpbmU9bi5URVhUX0JBU0VMSU5FLHRoaXMuX2NsaXBSb3cocik7dmFyIGk9ITE7ITEhPT10aGlzLl9vcHRpb25zU2VydmljZS5vcHRpb25zLmN1c3RvbUdseXBocyYmKGk9KDAsdS50cnlEcmF3Q3VzdG9tQ2hhcikodGhpcy5fY3R4LGUuZ2V0Q2hhcnMoKSx0KnRoaXMuX3NjYWxlZENlbGxXaWR0aCxyKnRoaXMuX3NjYWxlZENlbGxIZWlnaHQsdGhpcy5fc2NhbGVkQ2VsbFdpZHRoLHRoaXMuX3NjYWxlZENlbGxIZWlnaHQpKSxpfHx0aGlzLl9jdHguZmlsbFRleHQoZS5nZXRDaGFycygpLHQqdGhpcy5fc2NhbGVkQ2VsbFdpZHRoK3RoaXMuX3NjYWxlZENoYXJMZWZ0LHIqdGhpcy5fc2NhbGVkQ2VsbEhlaWdodCt0aGlzLl9zY2FsZWRDaGFyVG9wK3RoaXMuX3NjYWxlZENoYXJIZWlnaHQpfSxlLnByb3RvdHlwZS5fZHJhd0NoYXJzPWZ1bmN0aW9uKGUsdCxyKXt2YXIgbyxzLGEsYz10aGlzLl9nZXRDb250cmFzdENvbG9yKGUpO2N8fGUuaXNGZ1JHQigpfHxlLmlzQmdSR0IoKT90aGlzLl9kcmF3VW5jYWNoZWRDaGFycyhlLHQscixjKTooZS5pc0ludmVyc2UoKT8ocz1lLmlzQmdEZWZhdWx0KCk/bi5JTlZFUlRFRF9ERUZBVUxUX0NPTE9SOmUuZ2V0QmdDb2xvcigpLGE9ZS5pc0ZnRGVmYXVsdCgpP24uSU5WRVJURURfREVGQVVMVF9DT0xPUjplLmdldEZnQ29sb3IoKSk6KGE9ZS5pc0JnRGVmYXVsdCgpP2kuREVGQVVMVF9DT0xPUjplLmdldEJnQ29sb3IoKSxzPWUuaXNGZ0RlZmF1bHQoKT9pLkRFRkFVTFRfQ09MT1I6ZS5nZXRGZ0NvbG9yKCkpLHMrPXRoaXMuX29wdGlvbnNTZXJ2aWNlLm9wdGlvbnMuZHJhd0JvbGRUZXh0SW5CcmlnaHRDb2xvcnMmJmUuaXNCb2xkKCkmJnM8OD84OjAsdGhpcy5fY3VycmVudEdseXBoSWRlbnRpZmllci5jaGFycz1lLmdldENoYXJzKCl8fGkuV0hJVEVTUEFDRV9DRUxMX0NIQVIsdGhpcy5fY3VycmVudEdseXBoSWRlbnRpZmllci5jb2RlPWUuZ2V0Q29kZSgpfHxpLldISVRFU1BBQ0VfQ0VMTF9DT0RFLHRoaXMuX2N1cnJlbnRHbHlwaElkZW50aWZpZXIuYmc9YSx0aGlzLl9jdXJyZW50R2x5cGhJZGVudGlmaWVyLmZnPXMsdGhpcy5fY3VycmVudEdseXBoSWRlbnRpZmllci5ib2xkPSEhZS5pc0JvbGQoKSx0aGlzLl9jdXJyZW50R2x5cGhJZGVudGlmaWVyLmRpbT0hIWUuaXNEaW0oKSx0aGlzLl9jdXJyZW50R2x5cGhJZGVudGlmaWVyLml0YWxpYz0hIWUuaXNJdGFsaWMoKSwobnVsbD09PShvPXRoaXMuX2NoYXJBdGxhcyl8fHZvaWQgMD09PW8/dm9pZCAwOm8uZHJhdyh0aGlzLl9jdHgsdGhpcy5fY3VycmVudEdseXBoSWRlbnRpZmllcix0KnRoaXMuX3NjYWxlZENlbGxXaWR0aCt0aGlzLl9zY2FsZWRDaGFyTGVmdCxyKnRoaXMuX3NjYWxlZENlbGxIZWlnaHQrdGhpcy5fc2NhbGVkQ2hhclRvcCkpfHx0aGlzLl9kcmF3VW5jYWNoZWRDaGFycyhlLHQscikpfSxlLnByb3RvdHlwZS5fZHJhd1VuY2FjaGVkQ2hhcnM9ZnVuY3Rpb24oZSx0LHIsaSl7aWYodGhpcy5fY3R4LnNhdmUoKSx0aGlzLl9jdHguZm9udD10aGlzLl9nZXRGb250KCEhZS5pc0JvbGQoKSwhIWUuaXNJdGFsaWMoKSksdGhpcy5fY3R4LnRleHRCYXNlbGluZT1uLlRFWFRfQkFTRUxJTkUsZS5pc0ludmVyc2UoKSlpZihpKXRoaXMuX2N0eC5maWxsU3R5bGU9aS5jc3M7ZWxzZSBpZihlLmlzQmdEZWZhdWx0KCkpdGhpcy5fY3R4LmZpbGxTdHlsZT1jLmNvbG9yLm9wYXF1ZSh0aGlzLl9jb2xvcnMuYmFja2dyb3VuZCkuY3NzO2Vsc2UgaWYoZS5pc0JnUkdCKCkpdGhpcy5fY3R4LmZpbGxTdHlsZT0icmdiKCIrcy5BdHRyaWJ1dGVEYXRhLnRvQ29sb3JSR0IoZS5nZXRCZ0NvbG9yKCkpLmpvaW4oIiwiKSsiKSI7ZWxzZXt2YXIgbz1lLmdldEJnQ29sb3IoKTt0aGlzLl9vcHRpb25zU2VydmljZS5vcHRpb25zLmRyYXdCb2xkVGV4dEluQnJpZ2h0Q29sb3JzJiZlLmlzQm9sZCgpJiZvPDgmJihvKz04KSx0aGlzLl9jdHguZmlsbFN0eWxlPXRoaXMuX2NvbG9ycy5hbnNpW29dLmNzc31lbHNlIGlmKGkpdGhpcy5fY3R4LmZpbGxTdHlsZT1pLmNzcztlbHNlIGlmKGUuaXNGZ0RlZmF1bHQoKSl0aGlzLl9jdHguZmlsbFN0eWxlPXRoaXMuX2NvbG9ycy5mb3JlZ3JvdW5kLmNzcztlbHNlIGlmKGUuaXNGZ1JHQigpKXRoaXMuX2N0eC5maWxsU3R5bGU9InJnYigiK3MuQXR0cmlidXRlRGF0YS50b0NvbG9yUkdCKGUuZ2V0RmdDb2xvcigpKS5qb2luKCIsIikrIikiO2Vsc2V7dmFyIGE9ZS5nZXRGZ0NvbG9yKCk7dGhpcy5fb3B0aW9uc1NlcnZpY2Uub3B0aW9ucy5kcmF3Qm9sZFRleHRJbkJyaWdodENvbG9ycyYmZS5pc0JvbGQoKSYmYTw4JiYoYSs9OCksdGhpcy5fY3R4LmZpbGxTdHlsZT10aGlzLl9jb2xvcnMuYW5zaVthXS5jc3N9dGhpcy5fY2xpcFJvdyhyKSxlLmlzRGltKCkmJih0aGlzLl9jdHguZ2xvYmFsQWxwaGE9bi5ESU1fT1BBQ0lUWSk7dmFyIGw9ITE7ITEhPT10aGlzLl9vcHRpb25zU2VydmljZS5vcHRpb25zLmN1c3RvbUdseXBocyYmKGw9KDAsdS50cnlEcmF3Q3VzdG9tQ2hhcikodGhpcy5fY3R4LGUuZ2V0Q2hhcnMoKSx0KnRoaXMuX3NjYWxlZENlbGxXaWR0aCxyKnRoaXMuX3NjYWxlZENlbGxIZWlnaHQsdGhpcy5fc2NhbGVkQ2VsbFdpZHRoLHRoaXMuX3NjYWxlZENlbGxIZWlnaHQpKSxsfHx0aGlzLl9jdHguZmlsbFRleHQoZS5nZXRDaGFycygpLHQqdGhpcy5fc2NhbGVkQ2VsbFdpZHRoK3RoaXMuX3NjYWxlZENoYXJMZWZ0LHIqdGhpcy5fc2NhbGVkQ2VsbEhlaWdodCt0aGlzLl9zY2FsZWRDaGFyVG9wK3RoaXMuX3NjYWxlZENoYXJIZWlnaHQpLHRoaXMuX2N0eC5yZXN0b3JlKCl9LGUucHJvdG90eXBlLl9jbGlwUm93PWZ1bmN0aW9uKGUpe3RoaXMuX2N0eC5iZWdpblBhdGgoKSx0aGlzLl9jdHgucmVjdCgwLGUqdGhpcy5fc2NhbGVkQ2VsbEhlaWdodCx0aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHMqdGhpcy5fc2NhbGVkQ2VsbFdpZHRoLHRoaXMuX3NjYWxlZENlbGxIZWlnaHQpLHRoaXMuX2N0eC5jbGlwKCl9LGUucHJvdG90eXBlLl9nZXRGb250PWZ1bmN0aW9uKGUsdCl7cmV0dXJuKHQ/Iml0YWxpYyI6IiIpKyIgIisoZT90aGlzLl9vcHRpb25zU2VydmljZS5vcHRpb25zLmZvbnRXZWlnaHRCb2xkOnRoaXMuX29wdGlvbnNTZXJ2aWNlLm9wdGlvbnMuZm9udFdlaWdodCkrIiAiK3RoaXMuX29wdGlvbnNTZXJ2aWNlLm9wdGlvbnMuZm9udFNpemUqd2luZG93LmRldmljZVBpeGVsUmF0aW8rInB4ICIrdGhpcy5fb3B0aW9uc1NlcnZpY2Uub3B0aW9ucy5mb250RmFtaWx5fSxlLnByb3RvdHlwZS5fZ2V0Q29udHJhc3RDb2xvcj1mdW5jdGlvbihlKXtpZigxIT09dGhpcy5fb3B0aW9uc1NlcnZpY2Uub3B0aW9ucy5taW5pbXVtQ29udHJhc3RSYXRpbyl7dmFyIHQ9dGhpcy5fY29sb3JzLmNvbnRyYXN0Q2FjaGUuZ2V0Q29sb3IoZS5iZyxlLmZnKTtpZih2b2lkIDAhPT10KXJldHVybiB0fHx2b2lkIDA7dmFyIHI9ZS5nZXRGZ0NvbG9yKCksaT1lLmdldEZnQ29sb3JNb2RlKCksbj1lLmdldEJnQ29sb3IoKSxvPWUuZ2V0QmdDb2xvck1vZGUoKSxzPSEhZS5pc0ludmVyc2UoKSxhPSEhZS5pc0ludmVyc2UoKTtpZihzKXt2YXIgbD1yO3I9bixuPWw7dmFyIHU9aTtpPW8sbz11fXZhciBoPXRoaXMuX3Jlc29sdmVCYWNrZ3JvdW5kUmdiYShvLG4scyksZj10aGlzLl9yZXNvbHZlRm9yZWdyb3VuZFJnYmEoaSxyLHMsYSksXz1jLnJnYmEuZW5zdXJlQ29udHJhc3RSYXRpbyhoLGYsdGhpcy5fb3B0aW9uc1NlcnZpY2Uub3B0aW9ucy5taW5pbXVtQ29udHJhc3RSYXRpbyk7aWYoXyl7dmFyIGQ9e2NzczpjLmNoYW5uZWxzLnRvQ3NzKF8+PjI0JjI1NSxfPj4xNiYyNTUsXz4+OCYyNTUpLHJnYmE6X307cmV0dXJuIHRoaXMuX2NvbG9ycy5jb250cmFzdENhY2hlLnNldENvbG9yKGUuYmcsZS5mZyxkKSxkfXRoaXMuX2NvbG9ycy5jb250cmFzdENhY2hlLnNldENvbG9yKGUuYmcsZS5mZyxudWxsKX19LGUucHJvdG90eXBlLl9yZXNvbHZlQmFja2dyb3VuZFJnYmE9ZnVuY3Rpb24oZSx0LHIpe3N3aXRjaChlKXtjYXNlIDE2Nzc3MjE2OmNhc2UgMzM1NTQ0MzI6cmV0dXJuIHRoaXMuX2NvbG9ycy5hbnNpW3RdLnJnYmE7Y2FzZSA1MDMzMTY0ODpyZXR1cm4gdDw8ODtkZWZhdWx0OnJldHVybiByP3RoaXMuX2NvbG9ycy5mb3JlZ3JvdW5kLnJnYmE6dGhpcy5fY29sb3JzLmJhY2tncm91bmQucmdiYX19LGUucHJvdG90eXBlLl9yZXNvbHZlRm9yZWdyb3VuZFJnYmE9ZnVuY3Rpb24oZSx0LHIsaSl7c3dpdGNoKGUpe2Nhc2UgMTY3NzcyMTY6Y2FzZSAzMzU1NDQzMjpyZXR1cm4gdGhpcy5fb3B0aW9uc1NlcnZpY2Uub3B0aW9ucy5kcmF3Qm9sZFRleHRJbkJyaWdodENvbG9ycyYmaSYmdDw4JiYodCs9OCksdGhpcy5fY29sb3JzLmFuc2lbdF0ucmdiYTtjYXNlIDUwMzMxNjQ4OnJldHVybiB0PDw4O2RlZmF1bHQ6cmV0dXJuIHI/dGhpcy5fY29sb3JzLmJhY2tncm91bmQucmdiYTp0aGlzLl9jb2xvcnMuZm9yZWdyb3VuZC5yZ2JhfX0sZX0oKTt0LkJhc2VSZW5kZXJMYXllcj1ofSwyNTEyOmZ1bmN0aW9uKGUsdCxyKXt2YXIgaSxuPXRoaXMmJnRoaXMuX19leHRlbmRzfHwoaT1mdW5jdGlvbihlLHQpe3JldHVybiBpPU9iamVjdC5zZXRQcm90b3R5cGVPZnx8e19fcHJvdG9fXzpbXX1pbnN0YW5jZW9mIEFycmF5JiZmdW5jdGlvbihlLHQpe2UuX19wcm90b19fPXR9fHxmdW5jdGlvbihlLHQpe2Zvcih2YXIgciBpbiB0KU9iamVjdC5wcm90b3R5cGUuaGFzT3duUHJvcGVydHkuY2FsbCh0LHIpJiYoZVtyXT10W3JdKX0saShlLHQpfSxmdW5jdGlvbihlLHQpe2lmKCJmdW5jdGlvbiIhPXR5cGVvZiB0JiZudWxsIT09dCl0aHJvdyBuZXcgVHlwZUVycm9yKCJDbGFzcyBleHRlbmRzIHZhbHVlICIrU3RyaW5nKHQpKyIgaXMgbm90IGEgY29uc3RydWN0b3Igb3IgbnVsbCIpO2Z1bmN0aW9uIHIoKXt0aGlzLmNvbnN0cnVjdG9yPWV9aShlLHQpLGUucHJvdG90eXBlPW51bGw9PT10P09iamVjdC5jcmVhdGUodCk6KHIucHJvdG90eXBlPXQucHJvdG90eXBlLG5ldyByKX0pLG89dGhpcyYmdGhpcy5fX2RlY29yYXRlfHxmdW5jdGlvbihlLHQscixpKXt2YXIgbixvPWFyZ3VtZW50cy5sZW5ndGgscz1vPDM/dDpudWxsPT09aT9pPU9iamVjdC5nZXRPd25Qcm9wZXJ0eURlc2NyaXB0b3IodCxyKTppO2lmKCJvYmplY3QiPT10eXBlb2YgUmVmbGVjdCYmImZ1bmN0aW9uIj09dHlwZW9mIFJlZmxlY3QuZGVjb3JhdGUpcz1SZWZsZWN0LmRlY29yYXRlKGUsdCxyLGkpO2Vsc2UgZm9yKHZhciBhPWUubGVuZ3RoLTE7YT49MDthLS0pKG49ZVthXSkmJihzPShvPDM/bihzKTpvPjM/bih0LHIscyk6bih0LHIpKXx8cyk7cmV0dXJuIG8+MyYmcyYmT2JqZWN0LmRlZmluZVByb3BlcnR5KHQscixzKSxzfSxzPXRoaXMmJnRoaXMuX19wYXJhbXx8ZnVuY3Rpb24oZSx0KXtyZXR1cm4gZnVuY3Rpb24ocixpKXt0KHIsaSxlKX19O09iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LCJfX2VzTW9kdWxlIix7dmFsdWU6ITB9KSx0LkN1cnNvclJlbmRlckxheWVyPXZvaWQgMDt2YXIgYT1yKDE1NDYpLGM9cig1MTEpLGw9cigyNTg1KSx1PXIoNDcyNSksaD02MDAsZj1mdW5jdGlvbihlKXtmdW5jdGlvbiB0KHQscixpLG4sbyxzLGEsbCx1KXt2YXIgaD1lLmNhbGwodGhpcyx0LCJjdXJzb3IiLHIsITAsaSxuLHMsYSl8fHRoaXM7cmV0dXJuIGguX29uUmVxdWVzdFJlZHJhdz1vLGguX2NvcmVTZXJ2aWNlPWwsaC5fY29yZUJyb3dzZXJTZXJ2aWNlPXUsaC5fY2VsbD1uZXcgYy5DZWxsRGF0YSxoLl9zdGF0ZT17eDowLHk6MCxpc0ZvY3VzZWQ6ITEsc3R5bGU6IiIsd2lkdGg6MH0saC5fY3Vyc29yUmVuZGVyZXJzPXtiYXI6aC5fcmVuZGVyQmFyQ3Vyc29yLmJpbmQoaCksYmxvY2s6aC5fcmVuZGVyQmxvY2tDdXJzb3IuYmluZChoKSx1bmRlcmxpbmU6aC5fcmVuZGVyVW5kZXJsaW5lQ3Vyc29yLmJpbmQoaCl9LGh9cmV0dXJuIG4odCxlKSx0LnByb3RvdHlwZS5kaXNwb3NlPWZ1bmN0aW9uKCl7dGhpcy5fY3Vyc29yQmxpbmtTdGF0ZU1hbmFnZXImJih0aGlzLl9jdXJzb3JCbGlua1N0YXRlTWFuYWdlci5kaXNwb3NlKCksdGhpcy5fY3Vyc29yQmxpbmtTdGF0ZU1hbmFnZXI9dm9pZCAwKSxlLnByb3RvdHlwZS5kaXNwb3NlLmNhbGwodGhpcyl9LHQucHJvdG90eXBlLnJlc2l6ZT1mdW5jdGlvbih0KXtlLnByb3RvdHlwZS5yZXNpemUuY2FsbCh0aGlzLHQpLHRoaXMuX3N0YXRlPXt4OjAseTowLGlzRm9jdXNlZDohMSxzdHlsZToiIix3aWR0aDowfX0sdC5wcm90b3R5cGUucmVzZXQ9ZnVuY3Rpb24oKXt2YXIgZTt0aGlzLl9jbGVhckN1cnNvcigpLG51bGw9PT0oZT10aGlzLl9jdXJzb3JCbGlua1N0YXRlTWFuYWdlcil8fHZvaWQgMD09PWV8fGUucmVzdGFydEJsaW5rQW5pbWF0aW9uKCksdGhpcy5vbk9wdGlvbnNDaGFuZ2VkKCl9LHQucHJvdG90eXBlLm9uQmx1cj1mdW5jdGlvbigpe3ZhciBlO251bGw9PT0oZT10aGlzLl9jdXJzb3JCbGlua1N0YXRlTWFuYWdlcil8fHZvaWQgMD09PWV8fGUucGF1c2UoKSx0aGlzLl9vblJlcXVlc3RSZWRyYXcuZmlyZSh7c3RhcnQ6dGhpcy5fYnVmZmVyU2VydmljZS5idWZmZXIueSxlbmQ6dGhpcy5fYnVmZmVyU2VydmljZS5idWZmZXIueX0pfSx0LnByb3RvdHlwZS5vbkZvY3VzPWZ1bmN0aW9uKCl7dmFyIGU7bnVsbD09PShlPXRoaXMuX2N1cnNvckJsaW5rU3RhdGVNYW5hZ2VyKXx8dm9pZCAwPT09ZXx8ZS5yZXN1bWUoKSx0aGlzLl9vblJlcXVlc3RSZWRyYXcuZmlyZSh7c3RhcnQ6dGhpcy5fYnVmZmVyU2VydmljZS5idWZmZXIueSxlbmQ6dGhpcy5fYnVmZmVyU2VydmljZS5idWZmZXIueX0pfSx0LnByb3RvdHlwZS5vbk9wdGlvbnNDaGFuZ2VkPWZ1bmN0aW9uKCl7dmFyIGUsdD10aGlzO3RoaXMuX29wdGlvbnNTZXJ2aWNlLm9wdGlvbnMuY3Vyc29yQmxpbms/dGhpcy5fY3Vyc29yQmxpbmtTdGF0ZU1hbmFnZXJ8fCh0aGlzLl9jdXJzb3JCbGlua1N0YXRlTWFuYWdlcj1uZXcgXyh0aGlzLl9jb3JlQnJvd3NlclNlcnZpY2UuaXNGb2N1c2VkLChmdW5jdGlvbigpe3QuX3JlbmRlcighMCl9KSkpOihudWxsPT09KGU9dGhpcy5fY3Vyc29yQmxpbmtTdGF0ZU1hbmFnZXIpfHx2b2lkIDA9PT1lfHxlLmRpc3Bvc2UoKSx0aGlzLl9jdXJzb3JCbGlua1N0YXRlTWFuYWdlcj12b2lkIDApLHRoaXMuX29uUmVxdWVzdFJlZHJhdy5maXJlKHtzdGFydDp0aGlzLl9idWZmZXJTZXJ2aWNlLmJ1ZmZlci55LGVuZDp0aGlzLl9idWZmZXJTZXJ2aWNlLmJ1ZmZlci55fSl9LHQucHJvdG90eXBlLm9uQ3Vyc29yTW92ZT1mdW5jdGlvbigpe3ZhciBlO251bGw9PT0oZT10aGlzLl9jdXJzb3JCbGlua1N0YXRlTWFuYWdlcil8fHZvaWQgMD09PWV8fGUucmVzdGFydEJsaW5rQW5pbWF0aW9uKCl9LHQucHJvdG90eXBlLm9uR3JpZENoYW5nZWQ9ZnVuY3Rpb24oZSx0KXshdGhpcy5fY3Vyc29yQmxpbmtTdGF0ZU1hbmFnZXJ8fHRoaXMuX2N1cnNvckJsaW5rU3RhdGVNYW5hZ2VyLmlzUGF1c2VkP3RoaXMuX3JlbmRlcighMSk6dGhpcy5fY3Vyc29yQmxpbmtTdGF0ZU1hbmFnZXIucmVzdGFydEJsaW5rQW5pbWF0aW9uKCl9LHQucHJvdG90eXBlLl9yZW5kZXI9ZnVuY3Rpb24oZSl7aWYodGhpcy5fY29yZVNlcnZpY2UuaXNDdXJzb3JJbml0aWFsaXplZCYmIXRoaXMuX2NvcmVTZXJ2aWNlLmlzQ3Vyc29ySGlkZGVuKXt2YXIgdD10aGlzLl9idWZmZXJTZXJ2aWNlLmJ1ZmZlci55YmFzZSt0aGlzLl9idWZmZXJTZXJ2aWNlLmJ1ZmZlci55LHI9dC10aGlzLl9idWZmZXJTZXJ2aWNlLmJ1ZmZlci55ZGlzcDtpZihyPDB8fHI+PXRoaXMuX2J1ZmZlclNlcnZpY2Uucm93cyl0aGlzLl9jbGVhckN1cnNvcigpO2Vsc2V7dmFyIGk9TWF0aC5taW4odGhpcy5fYnVmZmVyU2VydmljZS5idWZmZXIueCx0aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHMtMSk7aWYodGhpcy5fYnVmZmVyU2VydmljZS5idWZmZXIubGluZXMuZ2V0KHQpLmxvYWRDZWxsKGksdGhpcy5fY2VsbCksdm9pZCAwIT09dGhpcy5fY2VsbC5jb250ZW50KXtpZighdGhpcy5fY29yZUJyb3dzZXJTZXJ2aWNlLmlzRm9jdXNlZCl7dGhpcy5fY2xlYXJDdXJzb3IoKSx0aGlzLl9jdHguc2F2ZSgpLHRoaXMuX2N0eC5maWxsU3R5bGU9dGhpcy5fY29sb3JzLmN1cnNvci5jc3M7dmFyIG49dGhpcy5fb3B0aW9uc1NlcnZpY2Uub3B0aW9ucy5jdXJzb3JTdHlsZTtyZXR1cm4gbiYmImJsb2NrIiE9PW4/dGhpcy5fY3Vyc29yUmVuZGVyZXJzW25dKGkscix0aGlzLl9jZWxsKTp0aGlzLl9yZW5kZXJCbHVyQ3Vyc29yKGkscix0aGlzLl9jZWxsKSx0aGlzLl9jdHgucmVzdG9yZSgpLHRoaXMuX3N0YXRlLng9aSx0aGlzLl9zdGF0ZS55PXIsdGhpcy5fc3RhdGUuaXNGb2N1c2VkPSExLHRoaXMuX3N0YXRlLnN0eWxlPW4sdm9pZCh0aGlzLl9zdGF0ZS53aWR0aD10aGlzLl9jZWxsLmdldFdpZHRoKCkpfWlmKCF0aGlzLl9jdXJzb3JCbGlua1N0YXRlTWFuYWdlcnx8dGhpcy5fY3Vyc29yQmxpbmtTdGF0ZU1hbmFnZXIuaXNDdXJzb3JWaXNpYmxlKXtpZih0aGlzLl9zdGF0ZSl7aWYodGhpcy5fc3RhdGUueD09PWkmJnRoaXMuX3N0YXRlLnk9PT1yJiZ0aGlzLl9zdGF0ZS5pc0ZvY3VzZWQ9PT10aGlzLl9jb3JlQnJvd3NlclNlcnZpY2UuaXNGb2N1c2VkJiZ0aGlzLl9zdGF0ZS5zdHlsZT09PXRoaXMuX29wdGlvbnNTZXJ2aWNlLm9wdGlvbnMuY3Vyc29yU3R5bGUmJnRoaXMuX3N0YXRlLndpZHRoPT09dGhpcy5fY2VsbC5nZXRXaWR0aCgpKXJldHVybjt0aGlzLl9jbGVhckN1cnNvcigpfXRoaXMuX2N0eC5zYXZlKCksdGhpcy5fY3Vyc29yUmVuZGVyZXJzW3RoaXMuX29wdGlvbnNTZXJ2aWNlLm9wdGlvbnMuY3Vyc29yU3R5bGV8fCJibG9jayJdKGkscix0aGlzLl9jZWxsKSx0aGlzLl9jdHgucmVzdG9yZSgpLHRoaXMuX3N0YXRlLng9aSx0aGlzLl9zdGF0ZS55PXIsdGhpcy5fc3RhdGUuaXNGb2N1c2VkPSExLHRoaXMuX3N0YXRlLnN0eWxlPXRoaXMuX29wdGlvbnNTZXJ2aWNlLm9wdGlvbnMuY3Vyc29yU3R5bGUsdGhpcy5fc3RhdGUud2lkdGg9dGhpcy5fY2VsbC5nZXRXaWR0aCgpfWVsc2UgdGhpcy5fY2xlYXJDdXJzb3IoKX19fWVsc2UgdGhpcy5fY2xlYXJDdXJzb3IoKX0sdC5wcm90b3R5cGUuX2NsZWFyQ3Vyc29yPWZ1bmN0aW9uKCl7dGhpcy5fc3RhdGUmJih3aW5kb3cuZGV2aWNlUGl4ZWxSYXRpbzwxP3RoaXMuX2NsZWFyQWxsKCk6dGhpcy5fY2xlYXJDZWxscyh0aGlzLl9zdGF0ZS54LHRoaXMuX3N0YXRlLnksdGhpcy5fc3RhdGUud2lkdGgsMSksdGhpcy5fc3RhdGU9e3g6MCx5OjAsaXNGb2N1c2VkOiExLHN0eWxlOiIiLHdpZHRoOjB9KX0sdC5wcm90b3R5cGUuX3JlbmRlckJhckN1cnNvcj1mdW5jdGlvbihlLHQscil7dGhpcy5fY3R4LnNhdmUoKSx0aGlzLl9jdHguZmlsbFN0eWxlPXRoaXMuX2NvbG9ycy5jdXJzb3IuY3NzLHRoaXMuX2ZpbGxMZWZ0TGluZUF0Q2VsbChlLHQsdGhpcy5fb3B0aW9uc1NlcnZpY2Uub3B0aW9ucy5jdXJzb3JXaWR0aCksdGhpcy5fY3R4LnJlc3RvcmUoKX0sdC5wcm90b3R5cGUuX3JlbmRlckJsb2NrQ3Vyc29yPWZ1bmN0aW9uKGUsdCxyKXt0aGlzLl9jdHguc2F2ZSgpLHRoaXMuX2N0eC5maWxsU3R5bGU9dGhpcy5fY29sb3JzLmN1cnNvci5jc3MsdGhpcy5fZmlsbENlbGxzKGUsdCxyLmdldFdpZHRoKCksMSksdGhpcy5fY3R4LmZpbGxTdHlsZT10aGlzLl9jb2xvcnMuY3Vyc29yQWNjZW50LmNzcyx0aGlzLl9maWxsQ2hhclRydWVDb2xvcihyLGUsdCksdGhpcy5fY3R4LnJlc3RvcmUoKX0sdC5wcm90b3R5cGUuX3JlbmRlclVuZGVybGluZUN1cnNvcj1mdW5jdGlvbihlLHQscil7dGhpcy5fY3R4LnNhdmUoKSx0aGlzLl9jdHguZmlsbFN0eWxlPXRoaXMuX2NvbG9ycy5jdXJzb3IuY3NzLHRoaXMuX2ZpbGxCb3R0b21MaW5lQXRDZWxscyhlLHQpLHRoaXMuX2N0eC5yZXN0b3JlKCl9LHQucHJvdG90eXBlLl9yZW5kZXJCbHVyQ3Vyc29yPWZ1bmN0aW9uKGUsdCxyKXt0aGlzLl9jdHguc2F2ZSgpLHRoaXMuX2N0eC5zdHJva2VTdHlsZT10aGlzLl9jb2xvcnMuY3Vyc29yLmNzcyx0aGlzLl9zdHJva2VSZWN0QXRDZWxsKGUsdCxyLmdldFdpZHRoKCksMSksdGhpcy5fY3R4LnJlc3RvcmUoKX0sbyhbcyg1LGwuSUJ1ZmZlclNlcnZpY2UpLHMoNixsLklPcHRpb25zU2VydmljZSkscyg3LGwuSUNvcmVTZXJ2aWNlKSxzKDgsdS5JQ29yZUJyb3dzZXJTZXJ2aWNlKV0sdCl9KGEuQmFzZVJlbmRlckxheWVyKTt0LkN1cnNvclJlbmRlckxheWVyPWY7dmFyIF89ZnVuY3Rpb24oKXtmdW5jdGlvbiBlKGUsdCl7dGhpcy5fcmVuZGVyQ2FsbGJhY2s9dCx0aGlzLmlzQ3Vyc29yVmlzaWJsZT0hMCxlJiZ0aGlzLl9yZXN0YXJ0SW50ZXJ2YWwoKX1yZXR1cm4gT2JqZWN0LmRlZmluZVByb3BlcnR5KGUucHJvdG90eXBlLCJpc1BhdXNlZCIse2dldDpmdW5jdGlvbigpe3JldHVybiEodGhpcy5fYmxpbmtTdGFydFRpbWVvdXR8fHRoaXMuX2JsaW5rSW50ZXJ2YWwpfSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLGUucHJvdG90eXBlLmRpc3Bvc2U9ZnVuY3Rpb24oKXt0aGlzLl9ibGlua0ludGVydmFsJiYod2luZG93LmNsZWFySW50ZXJ2YWwodGhpcy5fYmxpbmtJbnRlcnZhbCksdGhpcy5fYmxpbmtJbnRlcnZhbD12b2lkIDApLHRoaXMuX2JsaW5rU3RhcnRUaW1lb3V0JiYod2luZG93LmNsZWFyVGltZW91dCh0aGlzLl9ibGlua1N0YXJ0VGltZW91dCksdGhpcy5fYmxpbmtTdGFydFRpbWVvdXQ9dm9pZCAwKSx0aGlzLl9hbmltYXRpb25GcmFtZSYmKHdpbmRvdy5jYW5jZWxBbmltYXRpb25GcmFtZSh0aGlzLl9hbmltYXRpb25GcmFtZSksdGhpcy5fYW5pbWF0aW9uRnJhbWU9dm9pZCAwKX0sZS5wcm90b3R5cGUucmVzdGFydEJsaW5rQW5pbWF0aW9uPWZ1bmN0aW9uKCl7dmFyIGU9dGhpczt0aGlzLmlzUGF1c2VkfHwodGhpcy5fYW5pbWF0aW9uVGltZVJlc3RhcnRlZD1EYXRlLm5vdygpLHRoaXMuaXNDdXJzb3JWaXNpYmxlPSEwLHRoaXMuX2FuaW1hdGlvbkZyYW1lfHwodGhpcy5fYW5pbWF0aW9uRnJhbWU9d2luZG93LnJlcXVlc3RBbmltYXRpb25GcmFtZSgoZnVuY3Rpb24oKXtlLl9yZW5kZXJDYWxsYmFjaygpLGUuX2FuaW1hdGlvbkZyYW1lPXZvaWQgMH0pKSkpfSxlLnByb3RvdHlwZS5fcmVzdGFydEludGVydmFsPWZ1bmN0aW9uKGUpe3ZhciB0PXRoaXM7dm9pZCAwPT09ZSYmKGU9aCksdGhpcy5fYmxpbmtJbnRlcnZhbCYmKHdpbmRvdy5jbGVhckludGVydmFsKHRoaXMuX2JsaW5rSW50ZXJ2YWwpLHRoaXMuX2JsaW5rSW50ZXJ2YWw9dm9pZCAwKSx0aGlzLl9ibGlua1N0YXJ0VGltZW91dD13aW5kb3cuc2V0VGltZW91dCgoZnVuY3Rpb24oKXtpZih0Ll9hbmltYXRpb25UaW1lUmVzdGFydGVkKXt2YXIgZT1oLShEYXRlLm5vdygpLXQuX2FuaW1hdGlvblRpbWVSZXN0YXJ0ZWQpO2lmKHQuX2FuaW1hdGlvblRpbWVSZXN0YXJ0ZWQ9dm9pZCAwLGU+MClyZXR1cm4gdm9pZCB0Ll9yZXN0YXJ0SW50ZXJ2YWwoZSl9dC5pc0N1cnNvclZpc2libGU9ITEsdC5fYW5pbWF0aW9uRnJhbWU9d2luZG93LnJlcXVlc3RBbmltYXRpb25GcmFtZSgoZnVuY3Rpb24oKXt0Ll9yZW5kZXJDYWxsYmFjaygpLHQuX2FuaW1hdGlvbkZyYW1lPXZvaWQgMH0pKSx0Ll9ibGlua0ludGVydmFsPXdpbmRvdy5zZXRJbnRlcnZhbCgoZnVuY3Rpb24oKXtpZih0Ll9hbmltYXRpb25UaW1lUmVzdGFydGVkKXt2YXIgZT1oLShEYXRlLm5vdygpLXQuX2FuaW1hdGlvblRpbWVSZXN0YXJ0ZWQpO3JldHVybiB0Ll9hbmltYXRpb25UaW1lUmVzdGFydGVkPXZvaWQgMCx2b2lkIHQuX3Jlc3RhcnRJbnRlcnZhbChlKX10LmlzQ3Vyc29yVmlzaWJsZT0hdC5pc0N1cnNvclZpc2libGUsdC5fYW5pbWF0aW9uRnJhbWU9d2luZG93LnJlcXVlc3RBbmltYXRpb25GcmFtZSgoZnVuY3Rpb24oKXt0Ll9yZW5kZXJDYWxsYmFjaygpLHQuX2FuaW1hdGlvbkZyYW1lPXZvaWQgMH0pKX0pLGgpfSksZSl9LGUucHJvdG90eXBlLnBhdXNlPWZ1bmN0aW9uKCl7dGhpcy5pc0N1cnNvclZpc2libGU9ITAsdGhpcy5fYmxpbmtJbnRlcnZhbCYmKHdpbmRvdy5jbGVhckludGVydmFsKHRoaXMuX2JsaW5rSW50ZXJ2YWwpLHRoaXMuX2JsaW5rSW50ZXJ2YWw9dm9pZCAwKSx0aGlzLl9ibGlua1N0YXJ0VGltZW91dCYmKHdpbmRvdy5jbGVhclRpbWVvdXQodGhpcy5fYmxpbmtTdGFydFRpbWVvdXQpLHRoaXMuX2JsaW5rU3RhcnRUaW1lb3V0PXZvaWQgMCksdGhpcy5fYW5pbWF0aW9uRnJhbWUmJih3aW5kb3cuY2FuY2VsQW5pbWF0aW9uRnJhbWUodGhpcy5fYW5pbWF0aW9uRnJhbWUpLHRoaXMuX2FuaW1hdGlvbkZyYW1lPXZvaWQgMCl9LGUucHJvdG90eXBlLnJlc3VtZT1mdW5jdGlvbigpe3RoaXMucGF1c2UoKSx0aGlzLl9hbmltYXRpb25UaW1lUmVzdGFydGVkPXZvaWQgMCx0aGlzLl9yZXN0YXJ0SW50ZXJ2YWwoKSx0aGlzLnJlc3RhcnRCbGlua0FuaW1hdGlvbigpfSxlfSgpfSw4OTc4OihlLHQscik9Pnt2YXIgaSxuLG8scyxhLGMsbCx1LGgsZixfLGQscCx2LGcseSxtLGIsUyxDLHcsTCxFLHgsQSxrLE0sUixULE8sQixELFAsSSxILGosRixXLFUscSxOLHosSyxWLEcsWSxYLFosSiwkLFEsZWUsdGUscmUsaWUsbmUsb2Usc2UsYWUsY2UsbGUsdWUsaGUsZmUsX2UsZGUscGUsdmUsZ2UseWUsbWUsYmUsU2UsQ2Usd2UsTGUsRWUseGUsQWUsa2UsTWUsUmUsVGUsT2UsQmUsRGUsUGUsSWUsSGUsamUsRmUsV2UsVWUscWUsTmUsemUsS2UsVmUsR2UsWWUsWGUsWmUsSmUsJGUsUWUsZXQsdHQscnQsaXQsbnQsb3Qsc3QsYXQsY3QsbHQsdXQsaHQsZnQsX3QsZHQscHQsdnQsZ3QseXQsbXQsYnQsU3QsQ3Q7T2JqZWN0LmRlZmluZVByb3BlcnR5KHQsIl9fZXNNb2R1bGUiLHt2YWx1ZTohMH0pLHQudHJ5RHJhd0N1c3RvbUNoYXI9dC5ib3hEcmF3aW5nRGVmaW5pdGlvbnM9dC5ibG9ja0VsZW1lbnREZWZpbml0aW9ucz12b2lkIDA7dmFyIHd0PXIoMTc1Mik7dC5ibG9ja0VsZW1lbnREZWZpbml0aW9ucz17IuKWgCI6W3t4OjAseTowLHc6OCxoOjR9XSwi4paBIjpbe3g6MCx5Ojcsdzo4LGg6MX1dLCLiloIiOlt7eDowLHk6Nix3OjgsaDoyfV0sIuKWgyI6W3t4OjAseTo1LHc6OCxoOjN9XSwi4paEIjpbe3g6MCx5OjQsdzo4LGg6NH1dLCLiloUiOlt7eDowLHk6Myx3OjgsaDo1fV0sIuKWhiI6W3t4OjAseToyLHc6OCxoOjZ9XSwi4paHIjpbe3g6MCx5OjEsdzo4LGg6N31dLCLilogiOlt7eDowLHk6MCx3OjgsaDo4fV0sIuKWiSI6W3t4OjAseTowLHc6NyxoOjh9XSwi4paKIjpbe3g6MCx5OjAsdzo2LGg6OH1dLCLilosiOlt7eDowLHk6MCx3OjUsaDo4fV0sIuKWjCI6W3t4OjAseTowLHc6NCxoOjh9XSwi4paNIjpbe3g6MCx5OjAsdzozLGg6OH1dLCLilo4iOlt7eDowLHk6MCx3OjIsaDo4fV0sIuKWjyI6W3t4OjAseTowLHc6MSxoOjh9XSwi4paQIjpbe3g6NCx5OjAsdzo0LGg6OH1dLCLilpQiOlt7eDowLHk6MCx3OjksaDoxfV0sIuKWlSI6W3t4OjcseTowLHc6MSxoOjh9XSwi4paWIjpbe3g6MCx5OjQsdzo0LGg6NH1dLCLilpciOlt7eDo0LHk6NCx3OjQsaDo0fV0sIuKWmCI6W3t4OjAseTowLHc6NCxoOjR9XSwi4paZIjpbe3g6MCx5OjAsdzo0LGg6OH0se3g6MCx5OjQsdzo4LGg6NH1dLCLilpoiOlt7eDowLHk6MCx3OjQsaDo0fSx7eDo0LHk6NCx3OjQsaDo0fV0sIuKWmyI6W3t4OjAseTowLHc6NCxoOjh9LHt4OjAseTowLHc6NCxoOjh9XSwi4pacIjpbe3g6MCx5OjAsdzo4LGg6NH0se3g6NCx5OjAsdzo0LGg6OH1dLCLilp0iOlt7eDo0LHk6MCx3OjQsaDo0fV0sIuKWniI6W3t4OjQseTowLHc6NCxoOjR9LHt4OjAseTo0LHc6NCxoOjR9XSwi4pafIjpbe3g6NCx5OjAsdzo0LGg6OH0se3g6MCx5OjQsdzo4LGg6NH1dLCLwn62wIjpbe3g6MSx5OjAsdzoxLGg6OH1dLCLwn62xIjpbe3g6Mix5OjAsdzoxLGg6OH1dLCLwn62yIjpbe3g6Myx5OjAsdzoxLGg6OH1dLCLwn62zIjpbe3g6NCx5OjAsdzoxLGg6OH1dLCLwn620Ijpbe3g6NSx5OjAsdzoxLGg6OH1dLCLwn621Ijpbe3g6Nix5OjAsdzoxLGg6OH1dLCLwn622Ijpbe3g6MCx5OjEsdzo4LGg6MX1dLCLwn623Ijpbe3g6MCx5OjIsdzo4LGg6MX1dLCLwn624Ijpbe3g6MCx5OjMsdzo4LGg6MX1dLCLwn625Ijpbe3g6MCx5OjQsdzo4LGg6MX1dLCLwn626Ijpbe3g6MCx5OjUsdzo4LGg6MX1dLCLwn627Ijpbe3g6MCx5OjYsdzo4LGg6MX1dLCLwn628Ijpbe3g6MCx5OjAsdzoxLGg6OH0se3g6MCx5Ojcsdzo4LGg6MX1dLCLwn629Ijpbe3g6MCx5OjAsdzoxLGg6OH0se3g6MCx5OjAsdzo4LGg6MX1dLCLwn62+Ijpbe3g6Nyx5OjAsdzoxLGg6OH0se3g6MCx5OjAsdzo4LGg6MX1dLCLwn62/Ijpbe3g6Nyx5OjAsdzoxLGg6OH0se3g6MCx5Ojcsdzo4LGg6MX1dLCLwn66AIjpbe3g6MCx5OjAsdzo4LGg6MX0se3g6MCx5Ojcsdzo4LGg6MX1dLCLwn66BIjpbe3g6MCx5OjAsdzo4LGg6MX0se3g6MCx5OjIsdzo4LGg6MX0se3g6MCx5OjQsdzo4LGg6MX0se3g6MCx5Ojcsdzo4LGg6MX1dLCLwn66CIjpbe3g6MCx5OjAsdzo4LGg6Mn1dLCLwn66DIjpbe3g6MCx5OjAsdzo4LGg6M31dLCLwn66EIjpbe3g6MCx5OjAsdzo4LGg6NX1dLCLwn66FIjpbe3g6MCx5OjAsdzo4LGg6Nn1dLCLwn66GIjpbe3g6MCx5OjAsdzo4LGg6N31dLCLwn66HIjpbe3g6Nix5OjAsdzoyLGg6OH1dLCLwn66IIjpbe3g6NSx5OjAsdzozLGg6OH1dLCLwn66JIjpbe3g6Myx5OjAsdzo1LGg6OH1dLCLwn66KIjpbe3g6Mix5OjAsdzo2LGg6OH1dLCLwn66LIjpbe3g6MSx5OjAsdzo3LGg6OH1dLCLwn66VIjpbe3g6MCx5OjAsdzoyLGg6Mn0se3g6NCx5OjAsdzoyLGg6Mn0se3g6Mix5OjIsdzoyLGg6Mn0se3g6Nix5OjIsdzoyLGg6Mn0se3g6MCx5OjQsdzoyLGg6Mn0se3g6NCx5OjQsdzoyLGg6Mn0se3g6Mix5OjYsdzoyLGg6Mn0se3g6Nix5OjYsdzoyLGg6Mn1dLCLwn66WIjpbe3g6Mix5OjAsdzoyLGg6Mn0se3g6Nix5OjAsdzoyLGg6Mn0se3g6MCx5OjIsdzoyLGg6Mn0se3g6NCx5OjIsdzoyLGg6Mn0se3g6Mix5OjQsdzoyLGg6Mn0se3g6Nix5OjQsdzoyLGg6Mn0se3g6MCx5OjYsdzoyLGg6Mn0se3g6NCx5OjYsdzoyLGg6Mn1dLCLwn66XIjpbe3g6MCx5OjIsdzo4LGg6Mn0se3g6MCx5OjYsdzo4LGg6Mn1dfTt2YXIgTHQ9eyLilpEiOltbMSwwLDAsMF0sWzAsMCwwLDBdLFswLDAsMSwwXSxbMCwwLDAsMF1dLCLilpIiOltbMSwwXSxbMCwwXSxbMCwxXSxbMCwwXV0sIuKWkyI6W1swLDFdLFsxLDFdLFsxLDBdLFsxLDFdXX07dC5ib3hEcmF3aW5nRGVmaW5pdGlvbnM9eyLilIAiOihpPXt9LGlbMV09Ik0wLC41IEwxLC41IixpKSwi4pSBIjoobj17fSxuWzNdPSJNMCwuNSBMMSwuNSIsbiksIuKUgiI6KG89e30sb1sxXT0iTS41LDAgTC41LDEiLG8pLCLilIMiOihzPXt9LHNbM109Ik0uNSwwIEwuNSwxIixzKSwi4pSMIjooYT17fSxhWzFdPSJNMC41LDEgTC41LC41IEwxLC41IixhKSwi4pSPIjooYz17fSxjWzNdPSJNMC41LDEgTC41LC41IEwxLC41IixjKSwi4pSQIjoobD17fSxsWzFdPSJNMCwuNSBMLjUsLjUgTC41LDEiLGwpLCLilJMiOih1PXt9LHVbM109Ik0wLC41IEwuNSwuNSBMLjUsMSIsdSksIuKUlCI6KGg9e30saFsxXT0iTS41LDAgTC41LC41IEwxLC41IixoKSwi4pSXIjooZj17fSxmWzNdPSJNLjUsMCBMLjUsLjUgTDEsLjUiLGYpLCLilJgiOihfPXt9LF9bMV09Ik0uNSwwIEwuNSwuNSBMMCwuNSIsXyksIuKUmyI6KGQ9e30sZFszXT0iTS41LDAgTC41LC41IEwwLC41IixkKSwi4pScIjoocD17fSxwWzFdPSJNLjUsMCBMLjUsMSBNLjUsLjUgTDEsLjUiLHApLCLilKMiOih2PXt9LHZbM109Ik0uNSwwIEwuNSwxIE0uNSwuNSBMMSwuNSIsdiksIuKUpCI6KGc9e30sZ1sxXT0iTS41LDAgTC41LDEgTS41LC41IEwwLC41IixnKSwi4pSrIjooeT17fSx5WzNdPSJNLjUsMCBMLjUsMSBNLjUsLjUgTDAsLjUiLHkpLCLilKwiOihtPXt9LG1bMV09Ik0wLC41IEwxLC41IE0uNSwuNSBMLjUsMSIsbSksIuKUsyI6KGI9e30sYlszXT0iTTAsLjUgTDEsLjUgTS41LC41IEwuNSwxIixiKSwi4pS0IjooUz17fSxTWzFdPSJNMCwuNSBMMSwuNSBNLjUsLjUgTC41LDAiLFMpLCLilLsiOihDPXt9LENbM109Ik0wLC41IEwxLC41IE0uNSwuNSBMLjUsMCIsQyksIuKUvCI6KHc9e30sd1sxXT0iTTAsLjUgTDEsLjUgTS41LDAgTC41LDEiLHcpLCLilYsiOihMPXt9LExbM109Ik0wLC41IEwxLC41IE0uNSwwIEwuNSwxIixMKSwi4pW0IjooRT17fSxFWzFdPSJNLjUsLjUgTDAsLjUiLEUpLCLilbgiOih4PXt9LHhbM109Ik0uNSwuNSBMMCwuNSIseCksIuKVtSI6KEE9e30sQVsxXT0iTS41LC41IEwuNSwwIixBKSwi4pW5Ijooaz17fSxrWzNdPSJNLjUsLjUgTC41LDAiLGspLCLilbYiOihNPXt9LE1bMV09Ik0uNSwuNSBMMSwuNSIsTSksIuKVuiI6KFI9e30sUlszXT0iTS41LC41IEwxLC41IixSKSwi4pW3IjooVD17fSxUWzFdPSJNLjUsLjUgTC41LDEiLFQpLCLilbsiOihPPXt9LE9bM109Ik0uNSwuNSBMLjUsMSIsTyksIuKVkCI6KEI9e30sQlsxXT1mdW5jdGlvbihlLHQpe3JldHVybiJNMCwiKyguNS10KSsiIEwxLCIrKC41LXQpKyIgTTAsIisoLjUrdCkrIiBMMSwiKyguNSt0KX0sQiksIuKVkSI6KEQ9e30sRFsxXT1mdW5jdGlvbihlLHQpe3JldHVybiJNIisoLjUtZSkrIiwwIEwiKyguNS1lKSsiLDEgTSIrKC41K2UpKyIsMCBMIisoLjUrZSkrIiwxIn0sRCksIuKVkiI6KFA9e30sUFsxXT1mdW5jdGlvbihlLHQpe3JldHVybiJNLjUsMSBMLjUsIisoLjUtdCkrIiBMMSwiKyguNS10KSsiIE0uNSwiKyguNSt0KSsiIEwxLCIrKC41K3QpfSxQKSwi4pWTIjooST17fSxJWzFdPWZ1bmN0aW9uKGUsdCl7cmV0dXJuIk0iKyguNS1lKSsiLDEgTCIrKC41LWUpKyIsLjUgTDEsLjUgTSIrKC41K2UpKyIsLjUgTCIrKC41K2UpKyIsMSJ9LEkpLCLilZQiOihIPXt9LEhbMV09ZnVuY3Rpb24oZSx0KXtyZXR1cm4iTTEsIisoLjUtdCkrIiBMIisoLjUtZSkrIiwiKyguNS10KSsiIEwiKyguNS1lKSsiLDEgTTEsIisoLjUrdCkrIiBMIisoLjUrZSkrIiwiKyguNSt0KSsiIEwiKyguNStlKSsiLDEifSxIKSwi4pWVIjooaj17fSxqWzFdPWZ1bmN0aW9uKGUsdCl7cmV0dXJuIk0wLCIrKC41LXQpKyIgTC41LCIrKC41LXQpKyIgTC41LDEgTTAsIisoLjUrdCkrIiBMLjUsIisoLjUrdCl9LGopLCLilZYiOihGPXt9LEZbMV09ZnVuY3Rpb24oZSx0KXtyZXR1cm4iTSIrKC41K2UpKyIsMSBMIisoLjUrZSkrIiwuNSBMMCwuNSBNIisoLjUtZSkrIiwuNSBMIisoLjUtZSkrIiwxIn0sRiksIuKVlyI6KFc9e30sV1sxXT1mdW5jdGlvbihlLHQpe3JldHVybiJNMCwiKyguNSt0KSsiIEwiKyguNS1lKSsiLCIrKC41K3QpKyIgTCIrKC41LWUpKyIsMSBNMCwiKyguNS10KSsiIEwiKyguNStlKSsiLCIrKC41LXQpKyIgTCIrKC41K2UpKyIsMSJ9LFcpLCLilZgiOihVPXt9LFVbMV09ZnVuY3Rpb24oZSx0KXtyZXR1cm4iTS41LDAgTC41LCIrKC41K3QpKyIgTDEsIisoLjUrdCkrIiBNLjUsIisoLjUtdCkrIiBMMSwiKyguNS10KX0sVSksIuKVmSI6KHE9e30scVsxXT1mdW5jdGlvbihlLHQpe3JldHVybiJNMSwuNSBMIisoLjUtZSkrIiwuNSBMIisoLjUtZSkrIiwwIE0iKyguNStlKSsiLC41IEwiKyguNStlKSsiLDAifSxxKSwi4pWaIjooTj17fSxOWzFdPWZ1bmN0aW9uKGUsdCl7cmV0dXJuIk0xLCIrKC41LXQpKyIgTCIrKC41K2UpKyIsIisoLjUtdCkrIiBMIisoLjUrZSkrIiwwIE0xLCIrKC41K3QpKyIgTCIrKC41LWUpKyIsIisoLjUrdCkrIiBMIisoLjUtZSkrIiwwIn0sTiksIuKVmyI6KHo9e30selsxXT1mdW5jdGlvbihlLHQpe3JldHVybiJNMCwiKyguNSt0KSsiIEwuNSwiKyguNSt0KSsiIEwuNSwwIE0wLCIrKC41LXQpKyIgTC41LCIrKC41LXQpfSx6KSwi4pWcIjooSz17fSxLWzFdPWZ1bmN0aW9uKGUsdCl7cmV0dXJuIk0wLC41IEwiKyguNStlKSsiLC41IEwiKyguNStlKSsiLDAgTSIrKC41LWUpKyIsLjUgTCIrKC41LWUpKyIsMCJ9LEspLCLilZ0iOihWPXt9LFZbMV09ZnVuY3Rpb24oZSx0KXtyZXR1cm4iTTAsIisoLjUtdCkrIiBMIisoLjUtZSkrIiwiKyguNS10KSsiIEwiKyguNS1lKSsiLDAgTTAsIisoLjUrdCkrIiBMIisoLjUrZSkrIiwiKyguNSt0KSsiIEwiKyguNStlKSsiLDAifSxWKSwi4pWeIjooRz17fSxHWzFdPWZ1bmN0aW9uKGUsdCl7cmV0dXJuIk0uNSwwIEwuNSwxIE0uNSwiKyguNS10KSsiIEwxLCIrKC41LXQpKyIgTS41LCIrKC41K3QpKyIgTDEsIisoLjUrdCl9LEcpLCLilZ8iOihZPXt9LFlbMV09ZnVuY3Rpb24oZSx0KXtyZXR1cm4iTSIrKC41LWUpKyIsMCBMIisoLjUtZSkrIiwxIE0iKyguNStlKSsiLDAgTCIrKC41K2UpKyIsMSBNIisoLjUrZSkrIiwuNSBMMSwuNSJ9LFkpLCLilaAiOihYPXt9LFhbMV09ZnVuY3Rpb24oZSx0KXtyZXR1cm4iTSIrKC41LWUpKyIsMCBMIisoLjUtZSkrIiwxIE0xLCIrKC41K3QpKyIgTCIrKC41K2UpKyIsIisoLjUrdCkrIiBMIisoLjUrZSkrIiwxIE0xLCIrKC41LXQpKyIgTCIrKC41K2UpKyIsIisoLjUtdCkrIiBMIisoLjUrZSkrIiwwIn0sWCksIuKVoSI6KFo9e30sWlsxXT1mdW5jdGlvbihlLHQpe3JldHVybiJNLjUsMCBMLjUsMSBNMCwiKyguNS10KSsiIEwuNSwiKyguNS10KSsiIE0wLCIrKC41K3QpKyIgTC41LCIrKC41K3QpfSxaKSwi4pWiIjooSj17fSxKWzFdPWZ1bmN0aW9uKGUsdCl7cmV0dXJuIk0wLC41IEwiKyguNS1lKSsiLC41IE0iKyguNS1lKSsiLDAgTCIrKC41LWUpKyIsMSBNIisoLjUrZSkrIiwwIEwiKyguNStlKSsiLDEifSxKKSwi4pWjIjooJD17fSwkWzFdPWZ1bmN0aW9uKGUsdCl7cmV0dXJuIk0iKyguNStlKSsiLDAgTCIrKC41K2UpKyIsMSBNMCwiKyguNSt0KSsiIEwiKyguNS1lKSsiLCIrKC41K3QpKyIgTCIrKC41LWUpKyIsMSBNMCwiKyguNS10KSsiIEwiKyguNS1lKSsiLCIrKC41LXQpKyIgTCIrKC41LWUpKyIsMCJ9LCQpLCLilaQiOihRPXt9LFFbMV09ZnVuY3Rpb24oZSx0KXtyZXR1cm4iTTAsIisoLjUtdCkrIiBMMSwiKyguNS10KSsiIE0wLCIrKC41K3QpKyIgTDEsIisoLjUrdCkrIiBNLjUsIisoLjUrdCkrIiBMLjUsMSJ9LFEpLCLilaUiOihlZT17fSxlZVsxXT1mdW5jdGlvbihlLHQpe3JldHVybiJNMCwuNSBMMSwuNSBNIisoLjUtZSkrIiwuNSBMIisoLjUtZSkrIiwxIE0iKyguNStlKSsiLC41IEwiKyguNStlKSsiLDEifSxlZSksIuKVpiI6KHRlPXt9LHRlWzFdPWZ1bmN0aW9uKGUsdCl7cmV0dXJuIk0wLCIrKC41LXQpKyIgTDEsIisoLjUtdCkrIiBNMCwiKyguNSt0KSsiIEwiKyguNS1lKSsiLCIrKC41K3QpKyIgTCIrKC41LWUpKyIsMSBNMSwiKyguNSt0KSsiIEwiKyguNStlKSsiLCIrKC41K3QpKyIgTCIrKC41K2UpKyIsMSJ9LHRlKSwi4pWnIjoocmU9e30scmVbMV09ZnVuY3Rpb24oZSx0KXtyZXR1cm4iTS41LDAgTC41LCIrKC41LXQpKyIgTTAsIisoLjUtdCkrIiBMMSwiKyguNS10KSsiIE0wLCIrKC41K3QpKyIgTDEsIisoLjUrdCl9LHJlKSwi4pWoIjooaWU9e30saWVbMV09ZnVuY3Rpb24oZSx0KXtyZXR1cm4iTTAsLjUgTDEsLjUgTSIrKC41LWUpKyIsLjUgTCIrKC41LWUpKyIsMCBNIisoLjUrZSkrIiwuNSBMIisoLjUrZSkrIiwwIn0saWUpLCLilakiOihuZT17fSxuZVsxXT1mdW5jdGlvbihlLHQpe3JldHVybiJNMCwiKyguNSt0KSsiIEwxLCIrKC41K3QpKyIgTTAsIisoLjUtdCkrIiBMIisoLjUtZSkrIiwiKyguNS10KSsiIEwiKyguNS1lKSsiLDAgTTEsIisoLjUtdCkrIiBMIisoLjUrZSkrIiwiKyguNS10KSsiIEwiKyguNStlKSsiLDAifSxuZSksIuKVqiI6KG9lPXt9LG9lWzFdPWZ1bmN0aW9uKGUsdCl7cmV0dXJuIk0uNSwwIEwuNSwxIE0wLCIrKC41LXQpKyIgTDEsIisoLjUtdCkrIiBNMCwiKyguNSt0KSsiIEwxLCIrKC41K3QpfSxvZSksIuKVqyI6KHNlPXt9LHNlWzFdPWZ1bmN0aW9uKGUsdCl7cmV0dXJuIk0wLC41IEwxLC41IE0iKyguNS1lKSsiLDAgTCIrKC41LWUpKyIsMSBNIisoLjUrZSkrIiwwIEwiKyguNStlKSsiLDEifSxzZSksIuKVrCI6KGFlPXt9LGFlWzFdPWZ1bmN0aW9uKGUsdCl7cmV0dXJuIk0wLCIrKC41K3QpKyIgTCIrKC41LWUpKyIsIisoLjUrdCkrIiBMIisoLjUtZSkrIiwxIE0xLCIrKC41K3QpKyIgTCIrKC41K2UpKyIsIisoLjUrdCkrIiBMIisoLjUrZSkrIiwxIE0wLCIrKC41LXQpKyIgTCIrKC41LWUpKyIsIisoLjUtdCkrIiBMIisoLjUtZSkrIiwwIE0xLCIrKC41LXQpKyIgTCIrKC41K2UpKyIsIisoLjUtdCkrIiBMIisoLjUrZSkrIiwwIn0sYWUpLCLilbEiOihjZT17fSxjZVsxXT0iTTEsMCBMMCwxIixjZSksIuKVsiI6KGxlPXt9LGxlWzFdPSJNMCwwIEwxLDEiLGxlKSwi4pWzIjoodWU9e30sdWVbMV09Ik0xLDAgTDAsMSBNMCwwIEwxLDEiLHVlKSwi4pW8IjooaGU9e30saGVbMV09Ik0uNSwuNSBMMCwuNSIsaGVbM109Ik0uNSwuNSBMMSwuNSIsaGUpLCLilb0iOihmZT17fSxmZVsxXT0iTS41LC41IEwuNSwwIixmZVszXT0iTS41LC41IEwuNSwxIixmZSksIuKVviI6KF9lPXt9LF9lWzFdPSJNLjUsLjUgTDEsLjUiLF9lWzNdPSJNLjUsLjUgTDAsLjUiLF9lKSwi4pW/IjooZGU9e30sZGVbMV09Ik0uNSwuNSBMLjUsMSIsZGVbM109Ik0uNSwuNSBMLjUsMCIsZGUpLCLilI0iOihwZT17fSxwZVsxXT0iTS41LC41IEwuNSwxIixwZVszXT0iTS41LC41IEwxLC41IixwZSksIuKUjiI6KHZlPXt9LHZlWzFdPSJNLjUsLjUgTDEsLjUiLHZlWzNdPSJNLjUsLjUgTC41LDEiLHZlKSwi4pSRIjooZ2U9e30sZ2VbMV09Ik0uNSwuNSBMLjUsMSIsZ2VbM109Ik0uNSwuNSBMMCwuNSIsZ2UpLCLilJIiOih5ZT17fSx5ZVsxXT0iTS41LC41IEwwLC41Iix5ZVszXT0iTS41LC41IEwuNSwxIix5ZSksIuKUlSI6KG1lPXt9LG1lWzFdPSJNLjUsLjUgTC41LDAiLG1lWzNdPSJNLjUsLjUgTDEsLjUiLG1lKSwi4pSWIjooYmU9e30sYmVbMV09Ik0uNSwuNSBMMSwuNSIsYmVbM109Ik0uNSwuNSBMLjUsMCIsYmUpLCLilJkiOihTZT17fSxTZVsxXT0iTS41LC41IEwuNSwwIixTZVszXT0iTS41LC41IEwwLC41IixTZSksIuKUmiI6KENlPXt9LENlWzFdPSJNLjUsLjUgTDAsLjUiLENlWzNdPSJNLjUsLjUgTC41LDAiLENlKSwi4pSdIjood2U9e30sd2VbMV09Ik0uNSwwIEwuNSwxIix3ZVszXT0iTS41LC41IEwxLC41Iix3ZSksIuKUniI6KExlPXt9LExlWzFdPSJNMC41LDEgTC41LC41IEwxLC41IixMZVszXT0iTS41LC41IEwuNSwwIixMZSksIuKUnyI6KEVlPXt9LEVlWzFdPSJNLjUsMCBMLjUsLjUgTDEsLjUiLEVlWzNdPSJNLjUsLjUgTC41LDEiLEVlKSwi4pSgIjooeGU9e30seGVbMV09Ik0uNSwuNSBMMSwuNSIseGVbM109Ik0uNSwwIEwuNSwxIix4ZSksIuKUoSI6KEFlPXt9LEFlWzFdPSJNLjUsLjUgTC41LDEiLEFlWzNdPSJNLjUsMCBMLjUsLjUgTDEsLjUiLEFlKSwi4pSiIjooa2U9e30sa2VbMV09Ik0uNSwuNSBMLjUsMCIsa2VbM109Ik0wLjUsMSBMLjUsLjUgTDEsLjUiLGtlKSwi4pSlIjooTWU9e30sTWVbMV09Ik0uNSwwIEwuNSwxIixNZVszXT0iTS41LC41IEwwLC41IixNZSksIuKUpiI6KFJlPXt9LFJlWzFdPSJNMCwuNSBMLjUsLjUgTC41LDEiLFJlWzNdPSJNLjUsLjUgTC41LDAiLFJlKSwi4pSnIjooVGU9e30sVGVbMV09Ik0uNSwwIEwuNSwuNSBMMCwuNSIsVGVbM109Ik0uNSwuNSBMLjUsMSIsVGUpLCLilKgiOihPZT17fSxPZVsxXT0iTS41LC41IEwwLC41IixPZVszXT0iTS41LDAgTC41LDEiLE9lKSwi4pSpIjooQmU9e30sQmVbMV09Ik0uNSwuNSBMLjUsMSIsQmVbM109Ik0uNSwwIEwuNSwuNSBMMCwuNSIsQmUpLCLilKoiOihEZT17fSxEZVsxXT0iTS41LC41IEwuNSwwIixEZVszXT0iTTAsLjUgTC41LC41IEwuNSwxIixEZSksIuKUrSI6KFBlPXt9LFBlWzFdPSJNMC41LDEgTC41LC41IEwxLC41IixQZVszXT0iTS41LC41IEwwLC41IixQZSksIuKUriI6KEllPXt9LEllWzFdPSJNMCwuNSBMLjUsLjUgTC41LDEiLEllWzNdPSJNLjUsLjUgTDEsLjUiLEllKSwi4pSvIjooSGU9e30sSGVbMV09Ik0uNSwuNSBMLjUsMSIsSGVbM109Ik0wLC41IEwxLC41IixIZSksIuKUsCI6KGplPXt9LGplWzFdPSJNMCwuNSBMMSwuNSIsamVbM109Ik0uNSwuNSBMLjUsMSIsamUpLCLilLEiOihGZT17fSxGZVsxXT0iTS41LC41IEwxLC41IixGZVszXT0iTTAsLjUgTC41LC41IEwuNSwxIixGZSksIuKUsiI6KFdlPXt9LFdlWzFdPSJNLjUsLjUgTDAsLjUiLFdlWzNdPSJNMC41LDEgTC41LC41IEwxLC41IixXZSksIuKUtSI6KFVlPXt9LFVlWzFdPSJNLjUsMCBMLjUsLjUgTDEsLjUiLFVlWzNdPSJNLjUsLjUgTDAsLjUiLFVlKSwi4pS2IjoocWU9e30scWVbMV09Ik0uNSwwIEwuNSwuNSBMMCwuNSIscWVbM109Ik0uNSwuNSBMMSwuNSIscWUpLCLilLciOihOZT17fSxOZVsxXT0iTS41LC41IEwuNSwwIixOZVszXT0iTTAsLjUgTDEsLjUiLE5lKSwi4pS4IjooemU9e30semVbMV09Ik0wLC41IEwxLC41Iix6ZVszXT0iTS41LC41IEwuNSwwIix6ZSksIuKUuSI6KEtlPXt9LEtlWzFdPSJNLjUsLjUgTDEsLjUiLEtlWzNdPSJNLjUsMCBMLjUsLjUgTDAsLjUiLEtlKSwi4pS6IjooVmU9e30sVmVbMV09Ik0uNSwuNSBMMCwuNSIsVmVbM109Ik0uNSwwIEwuNSwuNSBMMSwuNSIsVmUpLCLilL0iOihHZT17fSxHZVsxXT0iTS41LDAgTC41LDEgTS41LC41IEwxLC41IixHZVszXT0iTS41LC41IEwwLC41IixHZSksIuKUviI6KFllPXt9LFllWzFdPSJNLjUsMCBMLjUsMSBNLjUsLjUgTDAsLjUiLFllWzNdPSJNLjUsLjUgTDEsLjUiLFllKSwi4pS/IjooWGU9e30sWGVbMV09Ik0uNSwwIEwuNSwxIixYZVszXT0iTTAsLjUgTDEsLjUiLFhlKSwi4pWAIjooWmU9e30sWmVbMV09Ik0wLC41IEwxLC41IE0uNSwuNSBMLjUsMSIsWmVbM109Ik0uNSwuNSBMLjUsMCIsWmUpLCLilYEiOihKZT17fSxKZVsxXT0iTS41LC41IEwuNSwwIE0wLC41IEwxLC41IixKZVszXT0iTS41LC41IEwuNSwxIixKZSksIuKVgiI6KCRlPXt9LCRlWzFdPSJNMCwuNSBMMSwuNSIsJGVbM109Ik0uNSwwIEwuNSwxIiwkZSksIuKVgyI6KFFlPXt9LFFlWzFdPSJNMC41LDEgTC41LC41IEwxLC41IixRZVszXT0iTS41LDAgTC41LC41IEwwLC41IixRZSksIuKVhCI6KGV0PXt9LGV0WzFdPSJNMCwuNSBMLjUsLjUgTC41LDEiLGV0WzNdPSJNLjUsMCBMLjUsLjUgTDEsLjUiLGV0KSwi4pWFIjoodHQ9e30sdHRbMV09Ik0uNSwwIEwuNSwuNSBMMSwuNSIsdHRbM109Ik0wLC41IEwuNSwuNSBMLjUsMSIsdHQpLCLilYYiOihydD17fSxydFsxXT0iTS41LDAgTC41LC41IEwwLC41IixydFszXT0iTTAuNSwxIEwuNSwuNSBMMSwuNSIscnQpLCLilYciOihpdD17fSxpdFsxXT0iTS41LC41IEwuNSwxIixpdFszXT0iTS41LC41IEwuNSwwIE0wLC41IEwxLC41IixpdCksIuKViCI6KG50PXt9LG50WzFdPSJNLjUsLjUgTC41LDAiLG50WzNdPSJNMCwuNSBMMSwuNSBNLjUsLjUgTC41LDEiLG50KSwi4pWJIjoob3Q9e30sb3RbMV09Ik0uNSwuNSBMMSwuNSIsb3RbM109Ik0uNSwwIEwuNSwxIE0uNSwuNSBMMCwuNSIsb3QpLCLilYoiOihzdD17fSxzdFsxXT0iTS41LC41IEwwLC41IixzdFszXT0iTS41LDAgTC41LDEgTS41LC41IEwxLC41IixzdCksIuKVjCI6KGF0PXt9LGF0WzFdPSJNLjEsLjUgTC40LC41IE0uNiwuNSBMLjksLjUiLGF0KSwi4pWNIjooY3Q9e30sY3RbM109Ik0uMSwuNSBMLjQsLjUgTS42LC41IEwuOSwuNSIsY3QpLCLilIQiOihsdD17fSxsdFsxXT0iTS4wNjY3LC41IEwuMjY2NywuNSBNLjQsLjUgTC42LC41IE0uNzMzMywuNSBMLjkzMzMsLjUiLGx0KSwi4pSFIjoodXQ9e30sdXRbM109Ik0uMDY2NywuNSBMLjI2NjcsLjUgTS40LC41IEwuNiwuNSBNLjczMzMsLjUgTC45MzMzLC41Iix1dCksIuKUiCI6KGh0PXt9LGh0WzFdPSJNLjA1LC41IEwuMiwuNSBNLjMsLjUgTC40NSwuNSBNLjU1LC41IEwuNywuNSBNLjgsLjUgTC45NSwuNSIsaHQpLCLilIkiOihmdD17fSxmdFszXT0iTS4wNSwuNSBMLjIsLjUgTS4zLC41IEwuNDUsLjUgTS41NSwuNSBMLjcsLjUgTS44LC41IEwuOTUsLjUiLGZ0KSwi4pWOIjooX3Q9e30sX3RbMV09Ik0uNSwuMSBMLjUsLjQgTS41LC42IEwuNSwuOSIsX3QpLCLilY8iOihkdD17fSxkdFszXT0iTS41LC4xIEwuNSwuNCBNLjUsLjYgTC41LC45IixkdCksIuKUhiI6KHB0PXt9LHB0WzFdPSJNLjUsLjA2NjcgTC41LC4yNjY3IE0uNSwuNCBMLjUsLjYgTS41LC43MzMzIEwuNSwuOTMzMyIscHQpLCLilIciOih2dD17fSx2dFszXT0iTS41LC4wNjY3IEwuNSwuMjY2NyBNLjUsLjQgTC41LC42IE0uNSwuNzMzMyBMLjUsLjkzMzMiLHZ0KSwi4pSKIjooZ3Q9e30sZ3RbMV09Ik0uNSwuMDUgTC41LC4yIE0uNSwuMyBMLjUsLjQ1IEwuNSwuNTUgTS41LC43IEwuNSwuOTUiLGd0KSwi4pSLIjooeXQ9e30seXRbM109Ik0uNSwuMDUgTC41LC4yIE0uNSwuMyBMLjUsLjQ1IEwuNSwuNTUgTS41LC43IEwuNSwuOTUiLHl0KSwi4pWtIjoobXQ9e30sbXRbMV09IkMuNSwxLC41LC41LDEsLjUiLG10KSwi4pWuIjooYnQ9e30sYnRbMV09IkMuNSwxLC41LC41LDAsLjUiLGJ0KSwi4pWvIjooU3Q9e30sU3RbMV09IkMuNSwwLC41LC41LDAsLjUiLFN0KSwi4pWwIjooQ3Q9e30sQ3RbMV09IkMuNSwwLC41LC41LDEsLjUiLEN0KX0sdC50cnlEcmF3Q3VzdG9tQ2hhcj1mdW5jdGlvbihlLHIsaSxuLG8scyl7dmFyIGE9dC5ibG9ja0VsZW1lbnREZWZpbml0aW9uc1tyXTtpZihhKXJldHVybiBmdW5jdGlvbihlLHQscixpLG4sbyl7Zm9yKHZhciBzPTA7czx0Lmxlbmd0aDtzKyspe3ZhciBhPXRbc10sYz1uLzgsbD1vLzg7ZS5maWxsUmVjdChyK2EueCpjLGkrYS55KmwsYS53KmMsYS5oKmwpfX0oZSxhLGksbixvLHMpLCEwO3ZhciBjPUx0W3JdO2lmKGMpcmV0dXJuIGZ1bmN0aW9uKGUsdCxyLGksbixvKXt2YXIgcyxhPUV0LmdldCh0KTthfHwoYT1uZXcgTWFwLEV0LnNldCh0LGEpKTt2YXIgYz1lLmZpbGxTdHlsZTtpZigic3RyaW5nIiE9dHlwZW9mIGMpdGhyb3cgbmV3IEVycm9yKCdVbmV4cGVjdGVkIGZpbGxTdHlsZSB0eXBlICInK2MrJyInKTt2YXIgbD1hLmdldChjKTtpZighbCl7dmFyIHU9dFswXS5sZW5ndGgsaD10Lmxlbmd0aCxmPWRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoImNhbnZhcyIpO2Yud2lkdGg9dSxmLmhlaWdodD1oO3ZhciBfPSgwLHd0LnRocm93SWZGYWxzeSkoZi5nZXRDb250ZXh0KCIyZCIpKSxkPW5ldyBJbWFnZURhdGEodSxoKSxwPXZvaWQgMCx2PXZvaWQgMCxnPXZvaWQgMCx5PXZvaWQgMDtpZihjLnN0YXJ0c1dpdGgoIiMiKSlwPXBhcnNlSW50KGMuc3Vic3RyKDEsMiksMTYpLHY9cGFyc2VJbnQoYy5zdWJzdHIoMywyKSwxNiksZz1wYXJzZUludChjLnN1YnN0cig1LDIpLDE2KSx5PWMubGVuZ3RoPjcmJnBhcnNlSW50KGMuc3Vic3RyKDcsMiksMTYpfHwxO2Vsc2V7aWYoIWMuc3RhcnRzV2l0aCgicmdiYSIpKXRocm93IG5ldyBFcnJvcignVW5leHBlY3RlZCBmaWxsU3R5bGUgY29sb3IgZm9ybWF0ICInK2MrJyIgd2hlbiBkcmF3aW5nIHBhdHRlcm4gZ2x5cGgnKTtwPShzPWMuc3Vic3RyaW5nKDUsYy5sZW5ndGgtMSkuc3BsaXQoIiwiKS5tYXAoKGZ1bmN0aW9uKGUpe3JldHVybiBwYXJzZUZsb2F0KGUpfSkpKVswXSx2PXNbMV0sZz1zWzJdLHk9c1szXX1mb3IodmFyIG09MDttPGg7bSsrKWZvcih2YXIgYj0wO2I8dTtiKyspZC5kYXRhWzQqKG0qdStiKV09cCxkLmRhdGFbNCoobSp1K2IpKzFdPXYsZC5kYXRhWzQqKG0qdStiKSsyXT1nLGQuZGF0YVs0KihtKnUrYikrM109dFttXVtiXSooMjU1KnkpO18ucHV0SW1hZ2VEYXRhKGQsMCwwKSxsPSgwLHd0LnRocm93SWZGYWxzeSkoZS5jcmVhdGVQYXR0ZXJuKGYsbnVsbCkpLGEuc2V0KGMsbCl9ZS5maWxsU3R5bGU9bCxlLmZpbGxSZWN0KHIsaSxuLG8pfShlLGMsaSxuLG8scyksITA7dmFyIGw9dC5ib3hEcmF3aW5nRGVmaW5pdGlvbnNbcl07cmV0dXJuISFsJiYoZnVuY3Rpb24oZSx0LHIsaSxuLG8pe2Uuc3Ryb2tlU3R5bGU9ZS5maWxsU3R5bGU7Zm9yKHZhciBzPTAsYT1PYmplY3QuZW50cmllcyh0KTtzPGEubGVuZ3RoO3MrKyl7dmFyIGM9YVtzXSxsPWNbMF0sdT1jWzFdO2UuYmVnaW5QYXRoKCksZS5saW5lV2lkdGg9d2luZG93LmRldmljZVBpeGVsUmF0aW8qTnVtYmVyLnBhcnNlSW50KGwpO2Zvcih2YXIgaD0wLGY9KCJmdW5jdGlvbiI9PXR5cGVvZiB1P3UoLjE1LC4xNS9vKm4pOnUpLnNwbGl0KCIgIik7aDxmLmxlbmd0aDtoKyspe3ZhciBfPWZbaF0sZD1fWzBdLHA9QXRbZF07aWYocCl7dmFyIHY9Xy5zdWJzdHJpbmcoMSkuc3BsaXQoIiwiKTt2WzBdJiZ2WzFdJiZwKGUsa3QodixuLG8scixpKSl9ZWxzZSBjb25zb2xlLmVycm9yKCdDb3VsZCBub3QgZmluZCBkcmF3aW5nIGluc3RydWN0aW9ucyBmb3IgIicrZCsnIicpfWUuc3Ryb2tlKCksZS5jbG9zZVBhdGgoKX19KGUsbCxpLG4sbyxzKSwhMCl9O3ZhciBFdD1uZXcgTWFwO2Z1bmN0aW9uIHh0KGUsdCxyKXtyZXR1cm4gdm9pZCAwPT09ciYmKHI9MCksTWF0aC5tYXgoTWF0aC5taW4oZSx0KSxyKX12YXIgQXQ9e0M6ZnVuY3Rpb24oZSx0KXtyZXR1cm4gZS5iZXppZXJDdXJ2ZVRvKHRbMF0sdFsxXSx0WzJdLHRbM10sdFs0XSx0WzVdKX0sTDpmdW5jdGlvbihlLHQpe3JldHVybiBlLmxpbmVUbyh0WzBdLHRbMV0pfSxNOmZ1bmN0aW9uKGUsdCl7cmV0dXJuIGUubW92ZVRvKHRbMF0sdFsxXSl9fTtmdW5jdGlvbiBrdChlLHQscixpLG4pe3ZhciBvPWUubWFwKChmdW5jdGlvbihlKXtyZXR1cm4gcGFyc2VGbG9hdChlKXx8cGFyc2VJbnQoZSl9KSk7aWYoby5sZW5ndGg8Mil0aHJvdyBuZXcgRXJyb3IoIlRvbyBmZXcgYXJndW1lbnRzIGZvciBpbnN0cnVjdGlvbiIpO2Zvcih2YXIgcz0wO3M8by5sZW5ndGg7cys9MilvW3NdKj10LDAhPT1vW3NdJiYob1tzXT14dChNYXRoLnJvdW5kKG9bc10rLjUpLS41LHQsMCkpLG9bc10rPWk7Zm9yKHZhciBhPTE7YTxvLmxlbmd0aDthKz0yKW9bYV0qPXIsMCE9PW9bYV0mJihvW2FdPXh0KE1hdGgucm91bmQob1thXSsuNSktLjUsciwwKSksb1thXSs9bjtyZXR1cm4gb319LDM3MDA6KGUsdCk9PntPYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC5HcmlkQ2FjaGU9dm9pZCAwO3ZhciByPWZ1bmN0aW9uKCl7ZnVuY3Rpb24gZSgpe3RoaXMuY2FjaGU9W119cmV0dXJuIGUucHJvdG90eXBlLnJlc2l6ZT1mdW5jdGlvbihlLHQpe2Zvcih2YXIgcj0wO3I8ZTtyKyspe3RoaXMuY2FjaGUubGVuZ3RoPD1yJiZ0aGlzLmNhY2hlLnB1c2goW10pO2Zvcih2YXIgaT10aGlzLmNhY2hlW3JdLmxlbmd0aDtpPHQ7aSsrKXRoaXMuY2FjaGVbcl0ucHVzaCh2b2lkIDApO3RoaXMuY2FjaGVbcl0ubGVuZ3RoPXR9dGhpcy5jYWNoZS5sZW5ndGg9ZX0sZS5wcm90b3R5cGUuY2xlYXI9ZnVuY3Rpb24oKXtmb3IodmFyIGU9MDtlPHRoaXMuY2FjaGUubGVuZ3RoO2UrKylmb3IodmFyIHQ9MDt0PHRoaXMuY2FjaGVbZV0ubGVuZ3RoO3QrKyl0aGlzLmNhY2hlW2VdW3RdPXZvaWQgMH0sZX0oKTt0LkdyaWRDYWNoZT1yfSw1MDk4OmZ1bmN0aW9uKGUsdCxyKXt2YXIgaSxuPXRoaXMmJnRoaXMuX19leHRlbmRzfHwoaT1mdW5jdGlvbihlLHQpe3JldHVybiBpPU9iamVjdC5zZXRQcm90b3R5cGVPZnx8e19fcHJvdG9fXzpbXX1pbnN0YW5jZW9mIEFycmF5JiZmdW5jdGlvbihlLHQpe2UuX19wcm90b19fPXR9fHxmdW5jdGlvbihlLHQpe2Zvcih2YXIgciBpbiB0KU9iamVjdC5wcm90b3R5cGUuaGFzT3duUHJvcGVydHkuY2FsbCh0LHIpJiYoZVtyXT10W3JdKX0saShlLHQpfSxmdW5jdGlvbihlLHQpe2lmKCJmdW5jdGlvbiIhPXR5cGVvZiB0JiZudWxsIT09dCl0aHJvdyBuZXcgVHlwZUVycm9yKCJDbGFzcyBleHRlbmRzIHZhbHVlICIrU3RyaW5nKHQpKyIgaXMgbm90IGEgY29uc3RydWN0b3Igb3IgbnVsbCIpO2Z1bmN0aW9uIHIoKXt0aGlzLmNvbnN0cnVjdG9yPWV9aShlLHQpLGUucHJvdG90eXBlPW51bGw9PT10P09iamVjdC5jcmVhdGUodCk6KHIucHJvdG90eXBlPXQucHJvdG90eXBlLG5ldyByKX0pLG89dGhpcyYmdGhpcy5fX2RlY29yYXRlfHxmdW5jdGlvbihlLHQscixpKXt2YXIgbixvPWFyZ3VtZW50cy5sZW5ndGgscz1vPDM/dDpudWxsPT09aT9pPU9iamVjdC5nZXRPd25Qcm9wZXJ0eURlc2NyaXB0b3IodCxyKTppO2lmKCJvYmplY3QiPT10eXBlb2YgUmVmbGVjdCYmImZ1bmN0aW9uIj09dHlwZW9mIFJlZmxlY3QuZGVjb3JhdGUpcz1SZWZsZWN0LmRlY29yYXRlKGUsdCxyLGkpO2Vsc2UgZm9yKHZhciBhPWUubGVuZ3RoLTE7YT49MDthLS0pKG49ZVthXSkmJihzPShvPDM/bihzKTpvPjM/bih0LHIscyk6bih0LHIpKXx8cyk7cmV0dXJuIG8+MyYmcyYmT2JqZWN0LmRlZmluZVByb3BlcnR5KHQscixzKSxzfSxzPXRoaXMmJnRoaXMuX19wYXJhbXx8ZnVuY3Rpb24oZSx0KXtyZXR1cm4gZnVuY3Rpb24ocixpKXt0KHIsaSxlKX19O09iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LCJfX2VzTW9kdWxlIix7dmFsdWU6ITB9KSx0LkxpbmtSZW5kZXJMYXllcj12b2lkIDA7dmFyIGE9cigxNTQ2KSxjPXIoODgwMyksbD1yKDIwNDApLHU9cigyNTg1KSxoPWZ1bmN0aW9uKGUpe2Z1bmN0aW9uIHQodCxyLGksbixvLHMsYSxjKXt2YXIgbD1lLmNhbGwodGhpcyx0LCJsaW5rIixyLCEwLGksbixhLGMpfHx0aGlzO3JldHVybiBvLm9uU2hvd0xpbmtVbmRlcmxpbmUoKGZ1bmN0aW9uKGUpe3JldHVybiBsLl9vblNob3dMaW5rVW5kZXJsaW5lKGUpfSkpLG8ub25IaWRlTGlua1VuZGVybGluZSgoZnVuY3Rpb24oZSl7cmV0dXJuIGwuX29uSGlkZUxpbmtVbmRlcmxpbmUoZSl9KSkscy5vblNob3dMaW5rVW5kZXJsaW5lKChmdW5jdGlvbihlKXtyZXR1cm4gbC5fb25TaG93TGlua1VuZGVybGluZShlKX0pKSxzLm9uSGlkZUxpbmtVbmRlcmxpbmUoKGZ1bmN0aW9uKGUpe3JldHVybiBsLl9vbkhpZGVMaW5rVW5kZXJsaW5lKGUpfSkpLGx9cmV0dXJuIG4odCxlKSx0LnByb3RvdHlwZS5yZXNpemU9ZnVuY3Rpb24odCl7ZS5wcm90b3R5cGUucmVzaXplLmNhbGwodGhpcyx0KSx0aGlzLl9zdGF0ZT12b2lkIDB9LHQucHJvdG90eXBlLnJlc2V0PWZ1bmN0aW9uKCl7dGhpcy5fY2xlYXJDdXJyZW50TGluaygpfSx0LnByb3RvdHlwZS5fY2xlYXJDdXJyZW50TGluaz1mdW5jdGlvbigpe2lmKHRoaXMuX3N0YXRlKXt0aGlzLl9jbGVhckNlbGxzKHRoaXMuX3N0YXRlLngxLHRoaXMuX3N0YXRlLnkxLHRoaXMuX3N0YXRlLmNvbHMtdGhpcy5fc3RhdGUueDEsMSk7dmFyIGU9dGhpcy5fc3RhdGUueTItdGhpcy5fc3RhdGUueTEtMTtlPjAmJnRoaXMuX2NsZWFyQ2VsbHMoMCx0aGlzLl9zdGF0ZS55MSsxLHRoaXMuX3N0YXRlLmNvbHMsZSksdGhpcy5fY2xlYXJDZWxscygwLHRoaXMuX3N0YXRlLnkyLHRoaXMuX3N0YXRlLngyLDEpLHRoaXMuX3N0YXRlPXZvaWQgMH19LHQucHJvdG90eXBlLl9vblNob3dMaW5rVW5kZXJsaW5lPWZ1bmN0aW9uKGUpe2lmKGUuZmc9PT1jLklOVkVSVEVEX0RFRkFVTFRfQ09MT1I/dGhpcy5fY3R4LmZpbGxTdHlsZT10aGlzLl9jb2xvcnMuYmFja2dyb3VuZC5jc3M6ZS5mZyYmKDAsbC5pczI1NkNvbG9yKShlLmZnKT90aGlzLl9jdHguZmlsbFN0eWxlPXRoaXMuX2NvbG9ycy5hbnNpW2UuZmddLmNzczp0aGlzLl9jdHguZmlsbFN0eWxlPXRoaXMuX2NvbG9ycy5mb3JlZ3JvdW5kLmNzcyxlLnkxPT09ZS55Mil0aGlzLl9maWxsQm90dG9tTGluZUF0Q2VsbHMoZS54MSxlLnkxLGUueDItZS54MSk7ZWxzZXt0aGlzLl9maWxsQm90dG9tTGluZUF0Q2VsbHMoZS54MSxlLnkxLGUuY29scy1lLngxKTtmb3IodmFyIHQ9ZS55MSsxO3Q8ZS55Mjt0KyspdGhpcy5fZmlsbEJvdHRvbUxpbmVBdENlbGxzKDAsdCxlLmNvbHMpO3RoaXMuX2ZpbGxCb3R0b21MaW5lQXRDZWxscygwLGUueTIsZS54Mil9dGhpcy5fc3RhdGU9ZX0sdC5wcm90b3R5cGUuX29uSGlkZUxpbmtVbmRlcmxpbmU9ZnVuY3Rpb24oZSl7dGhpcy5fY2xlYXJDdXJyZW50TGluaygpfSxvKFtzKDYsdS5JQnVmZmVyU2VydmljZSkscyg3LHUuSU9wdGlvbnNTZXJ2aWNlKV0sdCl9KGEuQmFzZVJlbmRlckxheWVyKTt0LkxpbmtSZW5kZXJMYXllcj1ofSwzNTI1OmZ1bmN0aW9uKGUsdCxyKXt2YXIgaSxuPXRoaXMmJnRoaXMuX19leHRlbmRzfHwoaT1mdW5jdGlvbihlLHQpe3JldHVybiBpPU9iamVjdC5zZXRQcm90b3R5cGVPZnx8e19fcHJvdG9fXzpbXX1pbnN0YW5jZW9mIEFycmF5JiZmdW5jdGlvbihlLHQpe2UuX19wcm90b19fPXR9fHxmdW5jdGlvbihlLHQpe2Zvcih2YXIgciBpbiB0KU9iamVjdC5wcm90b3R5cGUuaGFzT3duUHJvcGVydHkuY2FsbCh0LHIpJiYoZVtyXT10W3JdKX0saShlLHQpfSxmdW5jdGlvbihlLHQpe2lmKCJmdW5jdGlvbiIhPXR5cGVvZiB0JiZudWxsIT09dCl0aHJvdyBuZXcgVHlwZUVycm9yKCJDbGFzcyBleHRlbmRzIHZhbHVlICIrU3RyaW5nKHQpKyIgaXMgbm90IGEgY29uc3RydWN0b3Igb3IgbnVsbCIpO2Z1bmN0aW9uIHIoKXt0aGlzLmNvbnN0cnVjdG9yPWV9aShlLHQpLGUucHJvdG90eXBlPW51bGw9PT10P09iamVjdC5jcmVhdGUodCk6KHIucHJvdG90eXBlPXQucHJvdG90eXBlLG5ldyByKX0pLG89dGhpcyYmdGhpcy5fX2RlY29yYXRlfHxmdW5jdGlvbihlLHQscixpKXt2YXIgbixvPWFyZ3VtZW50cy5sZW5ndGgscz1vPDM/dDpudWxsPT09aT9pPU9iamVjdC5nZXRPd25Qcm9wZXJ0eURlc2NyaXB0b3IodCxyKTppO2lmKCJvYmplY3QiPT10eXBlb2YgUmVmbGVjdCYmImZ1bmN0aW9uIj09dHlwZW9mIFJlZmxlY3QuZGVjb3JhdGUpcz1SZWZsZWN0LmRlY29yYXRlKGUsdCxyLGkpO2Vsc2UgZm9yKHZhciBhPWUubGVuZ3RoLTE7YT49MDthLS0pKG49ZVthXSkmJihzPShvPDM/bihzKTpvPjM/bih0LHIscyk6bih0LHIpKXx8cyk7cmV0dXJuIG8+MyYmcyYmT2JqZWN0LmRlZmluZVByb3BlcnR5KHQscixzKSxzfSxzPXRoaXMmJnRoaXMuX19wYXJhbXx8ZnVuY3Rpb24oZSx0KXtyZXR1cm4gZnVuY3Rpb24ocixpKXt0KHIsaSxlKX19O09iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LCJfX2VzTW9kdWxlIix7dmFsdWU6ITB9KSx0LlJlbmRlcmVyPXZvaWQgMDt2YXIgYT1yKDk1OTYpLGM9cig0MTQ5KSxsPXIoMjUxMiksdT1yKDUwOTgpLGg9cig4NDQpLGY9cig0NzI1KSxfPXIoMjU4NSksZD1yKDE0MjApLHA9cig4NDYwKSx2PTEsZz1mdW5jdGlvbihlKXtmdW5jdGlvbiB0KHQscixpLG4sbyxzLGgsZil7dmFyIF89ZS5jYWxsKHRoaXMpfHx0aGlzO18uX2NvbG9ycz10LF8uX3NjcmVlbkVsZW1lbnQ9cixfLl9idWZmZXJTZXJ2aWNlPXMsXy5fY2hhclNpemVTZXJ2aWNlPWgsXy5fb3B0aW9uc1NlcnZpY2U9ZixfLl9pZD12KyssXy5fb25SZXF1ZXN0UmVkcmF3PW5ldyBwLkV2ZW50RW1pdHRlcjt2YXIgZD1fLl9vcHRpb25zU2VydmljZS5vcHRpb25zLmFsbG93VHJhbnNwYXJlbmN5O3JldHVybiBfLl9yZW5kZXJMYXllcnM9W28uY3JlYXRlSW5zdGFuY2UoYS5UZXh0UmVuZGVyTGF5ZXIsXy5fc2NyZWVuRWxlbWVudCwwLF8uX2NvbG9ycyxkLF8uX2lkKSxvLmNyZWF0ZUluc3RhbmNlKGMuU2VsZWN0aW9uUmVuZGVyTGF5ZXIsXy5fc2NyZWVuRWxlbWVudCwxLF8uX2NvbG9ycyxfLl9pZCksby5jcmVhdGVJbnN0YW5jZSh1LkxpbmtSZW5kZXJMYXllcixfLl9zY3JlZW5FbGVtZW50LDIsXy5fY29sb3JzLF8uX2lkLGksbiksby5jcmVhdGVJbnN0YW5jZShsLkN1cnNvclJlbmRlckxheWVyLF8uX3NjcmVlbkVsZW1lbnQsMyxfLl9jb2xvcnMsXy5faWQsXy5fb25SZXF1ZXN0UmVkcmF3KV0sXy5kaW1lbnNpb25zPXtzY2FsZWRDaGFyV2lkdGg6MCxzY2FsZWRDaGFySGVpZ2h0OjAsc2NhbGVkQ2VsbFdpZHRoOjAsc2NhbGVkQ2VsbEhlaWdodDowLHNjYWxlZENoYXJMZWZ0OjAsc2NhbGVkQ2hhclRvcDowLHNjYWxlZENhbnZhc1dpZHRoOjAsc2NhbGVkQ2FudmFzSGVpZ2h0OjAsY2FudmFzV2lkdGg6MCxjYW52YXNIZWlnaHQ6MCxhY3R1YWxDZWxsV2lkdGg6MCxhY3R1YWxDZWxsSGVpZ2h0OjB9LF8uX2RldmljZVBpeGVsUmF0aW89d2luZG93LmRldmljZVBpeGVsUmF0aW8sXy5fdXBkYXRlRGltZW5zaW9ucygpLF8ub25PcHRpb25zQ2hhbmdlZCgpLF99cmV0dXJuIG4odCxlKSxPYmplY3QuZGVmaW5lUHJvcGVydHkodC5wcm90b3R5cGUsIm9uUmVxdWVzdFJlZHJhdyIse2dldDpmdW5jdGlvbigpe3JldHVybiB0aGlzLl9vblJlcXVlc3RSZWRyYXcuZXZlbnR9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksdC5wcm90b3R5cGUuZGlzcG9zZT1mdW5jdGlvbigpe2Zvcih2YXIgdD0wLHI9dGhpcy5fcmVuZGVyTGF5ZXJzO3Q8ci5sZW5ndGg7dCsrKXJbdF0uZGlzcG9zZSgpO2UucHJvdG90eXBlLmRpc3Bvc2UuY2FsbCh0aGlzKSwoMCxkLnJlbW92ZVRlcm1pbmFsRnJvbUNhY2hlKSh0aGlzLl9pZCl9LHQucHJvdG90eXBlLm9uRGV2aWNlUGl4ZWxSYXRpb0NoYW5nZT1mdW5jdGlvbigpe3RoaXMuX2RldmljZVBpeGVsUmF0aW8hPT13aW5kb3cuZGV2aWNlUGl4ZWxSYXRpbyYmKHRoaXMuX2RldmljZVBpeGVsUmF0aW89d2luZG93LmRldmljZVBpeGVsUmF0aW8sdGhpcy5vblJlc2l6ZSh0aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHMsdGhpcy5fYnVmZmVyU2VydmljZS5yb3dzKSl9LHQucHJvdG90eXBlLnNldENvbG9ycz1mdW5jdGlvbihlKXt0aGlzLl9jb2xvcnM9ZTtmb3IodmFyIHQ9MCxyPXRoaXMuX3JlbmRlckxheWVyczt0PHIubGVuZ3RoO3QrKyl7dmFyIGk9clt0XTtpLnNldENvbG9ycyh0aGlzLl9jb2xvcnMpLGkucmVzZXQoKX19LHQucHJvdG90eXBlLm9uUmVzaXplPWZ1bmN0aW9uKGUsdCl7dGhpcy5fdXBkYXRlRGltZW5zaW9ucygpO2Zvcih2YXIgcj0wLGk9dGhpcy5fcmVuZGVyTGF5ZXJzO3I8aS5sZW5ndGg7cisrKWlbcl0ucmVzaXplKHRoaXMuZGltZW5zaW9ucyk7dGhpcy5fc2NyZWVuRWxlbWVudC5zdHlsZS53aWR0aD10aGlzLmRpbWVuc2lvbnMuY2FudmFzV2lkdGgrInB4Iix0aGlzLl9zY3JlZW5FbGVtZW50LnN0eWxlLmhlaWdodD10aGlzLmRpbWVuc2lvbnMuY2FudmFzSGVpZ2h0KyJweCJ9LHQucHJvdG90eXBlLm9uQ2hhclNpemVDaGFuZ2VkPWZ1bmN0aW9uKCl7dGhpcy5vblJlc2l6ZSh0aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHMsdGhpcy5fYnVmZmVyU2VydmljZS5yb3dzKX0sdC5wcm90b3R5cGUub25CbHVyPWZ1bmN0aW9uKCl7dGhpcy5fcnVuT3BlcmF0aW9uKChmdW5jdGlvbihlKXtyZXR1cm4gZS5vbkJsdXIoKX0pKX0sdC5wcm90b3R5cGUub25Gb2N1cz1mdW5jdGlvbigpe3RoaXMuX3J1bk9wZXJhdGlvbigoZnVuY3Rpb24oZSl7cmV0dXJuIGUub25Gb2N1cygpfSkpfSx0LnByb3RvdHlwZS5vblNlbGVjdGlvbkNoYW5nZWQ9ZnVuY3Rpb24oZSx0LHIpe3ZvaWQgMD09PXImJihyPSExKSx0aGlzLl9ydW5PcGVyYXRpb24oKGZ1bmN0aW9uKGkpe3JldHVybiBpLm9uU2VsZWN0aW9uQ2hhbmdlZChlLHQscil9KSl9LHQucHJvdG90eXBlLm9uQ3Vyc29yTW92ZT1mdW5jdGlvbigpe3RoaXMuX3J1bk9wZXJhdGlvbigoZnVuY3Rpb24oZSl7cmV0dXJuIGUub25DdXJzb3JNb3ZlKCl9KSl9LHQucHJvdG90eXBlLm9uT3B0aW9uc0NoYW5nZWQ9ZnVuY3Rpb24oKXt0aGlzLl9ydW5PcGVyYXRpb24oKGZ1bmN0aW9uKGUpe3JldHVybiBlLm9uT3B0aW9uc0NoYW5nZWQoKX0pKX0sdC5wcm90b3R5cGUuY2xlYXI9ZnVuY3Rpb24oKXt0aGlzLl9ydW5PcGVyYXRpb24oKGZ1bmN0aW9uKGUpe3JldHVybiBlLnJlc2V0KCl9KSl9LHQucHJvdG90eXBlLl9ydW5PcGVyYXRpb249ZnVuY3Rpb24oZSl7Zm9yKHZhciB0PTAscj10aGlzLl9yZW5kZXJMYXllcnM7dDxyLmxlbmd0aDt0KyspZShyW3RdKX0sdC5wcm90b3R5cGUucmVuZGVyUm93cz1mdW5jdGlvbihlLHQpe2Zvcih2YXIgcj0wLGk9dGhpcy5fcmVuZGVyTGF5ZXJzO3I8aS5sZW5ndGg7cisrKWlbcl0ub25HcmlkQ2hhbmdlZChlLHQpfSx0LnByb3RvdHlwZS5jbGVhclRleHR1cmVBdGxhcz1mdW5jdGlvbigpe2Zvcih2YXIgZT0wLHQ9dGhpcy5fcmVuZGVyTGF5ZXJzO2U8dC5sZW5ndGg7ZSsrKXRbZV0uY2xlYXJUZXh0dXJlQXRsYXMoKX0sdC5wcm90b3R5cGUuX3VwZGF0ZURpbWVuc2lvbnM9ZnVuY3Rpb24oKXt0aGlzLl9jaGFyU2l6ZVNlcnZpY2UuaGFzVmFsaWRTaXplJiYodGhpcy5kaW1lbnNpb25zLnNjYWxlZENoYXJXaWR0aD1NYXRoLmZsb29yKHRoaXMuX2NoYXJTaXplU2VydmljZS53aWR0aCp3aW5kb3cuZGV2aWNlUGl4ZWxSYXRpbyksdGhpcy5kaW1lbnNpb25zLnNjYWxlZENoYXJIZWlnaHQ9TWF0aC5jZWlsKHRoaXMuX2NoYXJTaXplU2VydmljZS5oZWlnaHQqd2luZG93LmRldmljZVBpeGVsUmF0aW8pLHRoaXMuZGltZW5zaW9ucy5zY2FsZWRDZWxsSGVpZ2h0PU1hdGguZmxvb3IodGhpcy5kaW1lbnNpb25zLnNjYWxlZENoYXJIZWlnaHQqdGhpcy5fb3B0aW9uc1NlcnZpY2Uub3B0aW9ucy5saW5lSGVpZ2h0KSx0aGlzLmRpbWVuc2lvbnMuc2NhbGVkQ2hhclRvcD0xPT09dGhpcy5fb3B0aW9uc1NlcnZpY2Uub3B0aW9ucy5saW5lSGVpZ2h0PzA6TWF0aC5yb3VuZCgodGhpcy5kaW1lbnNpb25zLnNjYWxlZENlbGxIZWlnaHQtdGhpcy5kaW1lbnNpb25zLnNjYWxlZENoYXJIZWlnaHQpLzIpLHRoaXMuZGltZW5zaW9ucy5zY2FsZWRDZWxsV2lkdGg9dGhpcy5kaW1lbnNpb25zLnNjYWxlZENoYXJXaWR0aCtNYXRoLnJvdW5kKHRoaXMuX29wdGlvbnNTZXJ2aWNlLm9wdGlvbnMubGV0dGVyU3BhY2luZyksdGhpcy5kaW1lbnNpb25zLnNjYWxlZENoYXJMZWZ0PU1hdGguZmxvb3IodGhpcy5fb3B0aW9uc1NlcnZpY2Uub3B0aW9ucy5sZXR0ZXJTcGFjaW5nLzIpLHRoaXMuZGltZW5zaW9ucy5zY2FsZWRDYW52YXNIZWlnaHQ9dGhpcy5fYnVmZmVyU2VydmljZS5yb3dzKnRoaXMuZGltZW5zaW9ucy5zY2FsZWRDZWxsSGVpZ2h0LHRoaXMuZGltZW5zaW9ucy5zY2FsZWRDYW52YXNXaWR0aD10aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHMqdGhpcy5kaW1lbnNpb25zLnNjYWxlZENlbGxXaWR0aCx0aGlzLmRpbWVuc2lvbnMuY2FudmFzSGVpZ2h0PU1hdGgucm91bmQodGhpcy5kaW1lbnNpb25zLnNjYWxlZENhbnZhc0hlaWdodC93aW5kb3cuZGV2aWNlUGl4ZWxSYXRpbyksdGhpcy5kaW1lbnNpb25zLmNhbnZhc1dpZHRoPU1hdGgucm91bmQodGhpcy5kaW1lbnNpb25zLnNjYWxlZENhbnZhc1dpZHRoL3dpbmRvdy5kZXZpY2VQaXhlbFJhdGlvKSx0aGlzLmRpbWVuc2lvbnMuYWN0dWFsQ2VsbEhlaWdodD10aGlzLmRpbWVuc2lvbnMuY2FudmFzSGVpZ2h0L3RoaXMuX2J1ZmZlclNlcnZpY2Uucm93cyx0aGlzLmRpbWVuc2lvbnMuYWN0dWFsQ2VsbFdpZHRoPXRoaXMuZGltZW5zaW9ucy5jYW52YXNXaWR0aC90aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHMpfSxvKFtzKDQsXy5JSW5zdGFudGlhdGlvblNlcnZpY2UpLHMoNSxfLklCdWZmZXJTZXJ2aWNlKSxzKDYsZi5JQ2hhclNpemVTZXJ2aWNlKSxzKDcsXy5JT3B0aW9uc1NlcnZpY2UpXSx0KX0oaC5EaXNwb3NhYmxlKTt0LlJlbmRlcmVyPWd9LDE3NTI6KGUsdCk9PntPYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC50aHJvd0lmRmFsc3k9dm9pZCAwLHQudGhyb3dJZkZhbHN5PWZ1bmN0aW9uKGUpe2lmKCFlKXRocm93IG5ldyBFcnJvcigidmFsdWUgbXVzdCBub3QgYmUgZmFsc3kiKTtyZXR1cm4gZX19LDQxNDk6ZnVuY3Rpb24oZSx0LHIpe3ZhciBpLG49dGhpcyYmdGhpcy5fX2V4dGVuZHN8fChpPWZ1bmN0aW9uKGUsdCl7cmV0dXJuIGk9T2JqZWN0LnNldFByb3RvdHlwZU9mfHx7X19wcm90b19fOltdfWluc3RhbmNlb2YgQXJyYXkmJmZ1bmN0aW9uKGUsdCl7ZS5fX3Byb3RvX189dH18fGZ1bmN0aW9uKGUsdCl7Zm9yKHZhciByIGluIHQpT2JqZWN0LnByb3RvdHlwZS5oYXNPd25Qcm9wZXJ0eS5jYWxsKHQscikmJihlW3JdPXRbcl0pfSxpKGUsdCl9LGZ1bmN0aW9uKGUsdCl7aWYoImZ1bmN0aW9uIiE9dHlwZW9mIHQmJm51bGwhPT10KXRocm93IG5ldyBUeXBlRXJyb3IoIkNsYXNzIGV4dGVuZHMgdmFsdWUgIitTdHJpbmcodCkrIiBpcyBub3QgYSBjb25zdHJ1Y3RvciBvciBudWxsIik7ZnVuY3Rpb24gcigpe3RoaXMuY29uc3RydWN0b3I9ZX1pKGUsdCksZS5wcm90b3R5cGU9bnVsbD09PXQ/T2JqZWN0LmNyZWF0ZSh0KTooci5wcm90b3R5cGU9dC5wcm90b3R5cGUsbmV3IHIpfSksbz10aGlzJiZ0aGlzLl9fZGVjb3JhdGV8fGZ1bmN0aW9uKGUsdCxyLGkpe3ZhciBuLG89YXJndW1lbnRzLmxlbmd0aCxzPW88Mz90Om51bGw9PT1pP2k9T2JqZWN0LmdldE93blByb3BlcnR5RGVzY3JpcHRvcih0LHIpOmk7aWYoIm9iamVjdCI9PXR5cGVvZiBSZWZsZWN0JiYiZnVuY3Rpb24iPT10eXBlb2YgUmVmbGVjdC5kZWNvcmF0ZSlzPVJlZmxlY3QuZGVjb3JhdGUoZSx0LHIsaSk7ZWxzZSBmb3IodmFyIGE9ZS5sZW5ndGgtMTthPj0wO2EtLSkobj1lW2FdKSYmKHM9KG88Mz9uKHMpOm8+Mz9uKHQscixzKTpuKHQscikpfHxzKTtyZXR1cm4gbz4zJiZzJiZPYmplY3QuZGVmaW5lUHJvcGVydHkodCxyLHMpLHN9LHM9dGhpcyYmdGhpcy5fX3BhcmFtfHxmdW5jdGlvbihlLHQpe3JldHVybiBmdW5jdGlvbihyLGkpe3QocixpLGUpfX07T2JqZWN0LmRlZmluZVByb3BlcnR5KHQsIl9fZXNNb2R1bGUiLHt2YWx1ZTohMH0pLHQuU2VsZWN0aW9uUmVuZGVyTGF5ZXI9dm9pZCAwO3ZhciBhPXIoMTU0NiksYz1yKDI1ODUpLGw9ZnVuY3Rpb24oZSl7ZnVuY3Rpb24gdCh0LHIsaSxuLG8scyl7dmFyIGE9ZS5jYWxsKHRoaXMsdCwic2VsZWN0aW9uIixyLCEwLGksbixvLHMpfHx0aGlzO3JldHVybiBhLl9jbGVhclN0YXRlKCksYX1yZXR1cm4gbih0LGUpLHQucHJvdG90eXBlLl9jbGVhclN0YXRlPWZ1bmN0aW9uKCl7dGhpcy5fc3RhdGU9e3N0YXJ0OnZvaWQgMCxlbmQ6dm9pZCAwLGNvbHVtblNlbGVjdE1vZGU6dm9pZCAwLHlkaXNwOnZvaWQgMH19LHQucHJvdG90eXBlLnJlc2l6ZT1mdW5jdGlvbih0KXtlLnByb3RvdHlwZS5yZXNpemUuY2FsbCh0aGlzLHQpLHRoaXMuX2NsZWFyU3RhdGUoKX0sdC5wcm90b3R5cGUucmVzZXQ9ZnVuY3Rpb24oKXt0aGlzLl9zdGF0ZS5zdGFydCYmdGhpcy5fc3RhdGUuZW5kJiYodGhpcy5fY2xlYXJTdGF0ZSgpLHRoaXMuX2NsZWFyQWxsKCkpfSx0LnByb3RvdHlwZS5vblNlbGVjdGlvbkNoYW5nZWQ9ZnVuY3Rpb24oZSx0LHIpe2lmKHRoaXMuX2RpZFN0YXRlQ2hhbmdlKGUsdCxyLHRoaXMuX2J1ZmZlclNlcnZpY2UuYnVmZmVyLnlkaXNwKSlpZih0aGlzLl9jbGVhckFsbCgpLGUmJnQpe3ZhciBpPWVbMV0tdGhpcy5fYnVmZmVyU2VydmljZS5idWZmZXIueWRpc3Asbj10WzFdLXRoaXMuX2J1ZmZlclNlcnZpY2UuYnVmZmVyLnlkaXNwLG89TWF0aC5tYXgoaSwwKSxzPU1hdGgubWluKG4sdGhpcy5fYnVmZmVyU2VydmljZS5yb3dzLTEpO2lmKG8+PXRoaXMuX2J1ZmZlclNlcnZpY2Uucm93c3x8czwwKXRoaXMuX3N0YXRlLnlkaXNwPXRoaXMuX2J1ZmZlclNlcnZpY2UuYnVmZmVyLnlkaXNwO2Vsc2V7aWYodGhpcy5fY3R4LmZpbGxTdHlsZT10aGlzLl9jb2xvcnMuc2VsZWN0aW9uVHJhbnNwYXJlbnQuY3NzLHIpe3ZhciBhPWVbMF0sYz10WzBdLWEsbD1zLW8rMTt0aGlzLl9maWxsQ2VsbHMoYSxvLGMsbCl9ZWxzZXthPWk9PT1vP2VbMF06MDt2YXIgdT1vPT09bj90WzBdOnRoaXMuX2J1ZmZlclNlcnZpY2UuY29sczt0aGlzLl9maWxsQ2VsbHMoYSxvLHUtYSwxKTt2YXIgaD1NYXRoLm1heChzLW8tMSwwKTtpZih0aGlzLl9maWxsQ2VsbHMoMCxvKzEsdGhpcy5fYnVmZmVyU2VydmljZS5jb2xzLGgpLG8hPT1zKXt2YXIgZj1uPT09cz90WzBdOnRoaXMuX2J1ZmZlclNlcnZpY2UuY29sczt0aGlzLl9maWxsQ2VsbHMoMCxzLGYsMSl9fXRoaXMuX3N0YXRlLnN0YXJ0PVtlWzBdLGVbMV1dLHRoaXMuX3N0YXRlLmVuZD1bdFswXSx0WzFdXSx0aGlzLl9zdGF0ZS5jb2x1bW5TZWxlY3RNb2RlPXIsdGhpcy5fc3RhdGUueWRpc3A9dGhpcy5fYnVmZmVyU2VydmljZS5idWZmZXIueWRpc3B9fWVsc2UgdGhpcy5fY2xlYXJTdGF0ZSgpfSx0LnByb3RvdHlwZS5fZGlkU3RhdGVDaGFuZ2U9ZnVuY3Rpb24oZSx0LHIsaSl7cmV0dXJuIXRoaXMuX2FyZUNvb3JkaW5hdGVzRXF1YWwoZSx0aGlzLl9zdGF0ZS5zdGFydCl8fCF0aGlzLl9hcmVDb29yZGluYXRlc0VxdWFsKHQsdGhpcy5fc3RhdGUuZW5kKXx8ciE9PXRoaXMuX3N0YXRlLmNvbHVtblNlbGVjdE1vZGV8fGkhPT10aGlzLl9zdGF0ZS55ZGlzcH0sdC5wcm90b3R5cGUuX2FyZUNvb3JkaW5hdGVzRXF1YWw9ZnVuY3Rpb24oZSx0KXtyZXR1cm4hKCFlfHwhdCkmJmVbMF09PT10WzBdJiZlWzFdPT09dFsxXX0sbyhbcyg0LGMuSUJ1ZmZlclNlcnZpY2UpLHMoNSxjLklPcHRpb25zU2VydmljZSldLHQpfShhLkJhc2VSZW5kZXJMYXllcik7dC5TZWxlY3Rpb25SZW5kZXJMYXllcj1sfSw5NTk2OmZ1bmN0aW9uKGUsdCxyKXt2YXIgaSxuPXRoaXMmJnRoaXMuX19leHRlbmRzfHwoaT1mdW5jdGlvbihlLHQpe3JldHVybiBpPU9iamVjdC5zZXRQcm90b3R5cGVPZnx8e19fcHJvdG9fXzpbXX1pbnN0YW5jZW9mIEFycmF5JiZmdW5jdGlvbihlLHQpe2UuX19wcm90b19fPXR9fHxmdW5jdGlvbihlLHQpe2Zvcih2YXIgciBpbiB0KU9iamVjdC5wcm90b3R5cGUuaGFzT3duUHJvcGVydHkuY2FsbCh0LHIpJiYoZVtyXT10W3JdKX0saShlLHQpfSxmdW5jdGlvbihlLHQpe2lmKCJmdW5jdGlvbiIhPXR5cGVvZiB0JiZudWxsIT09dCl0aHJvdyBuZXcgVHlwZUVycm9yKCJDbGFzcyBleHRlbmRzIHZhbHVlICIrU3RyaW5nKHQpKyIgaXMgbm90IGEgY29uc3RydWN0b3Igb3IgbnVsbCIpO2Z1bmN0aW9uIHIoKXt0aGlzLmNvbnN0cnVjdG9yPWV9aShlLHQpLGUucHJvdG90eXBlPW51bGw9PT10P09iamVjdC5jcmVhdGUodCk6KHIucHJvdG90eXBlPXQucHJvdG90eXBlLG5ldyByKX0pLG89dGhpcyYmdGhpcy5fX2RlY29yYXRlfHxmdW5jdGlvbihlLHQscixpKXt2YXIgbixvPWFyZ3VtZW50cy5sZW5ndGgscz1vPDM/dDpudWxsPT09aT9pPU9iamVjdC5nZXRPd25Qcm9wZXJ0eURlc2NyaXB0b3IodCxyKTppO2lmKCJvYmplY3QiPT10eXBlb2YgUmVmbGVjdCYmImZ1bmN0aW9uIj09dHlwZW9mIFJlZmxlY3QuZGVjb3JhdGUpcz1SZWZsZWN0LmRlY29yYXRlKGUsdCxyLGkpO2Vsc2UgZm9yKHZhciBhPWUubGVuZ3RoLTE7YT49MDthLS0pKG49ZVthXSkmJihzPShvPDM/bihzKTpvPjM/bih0LHIscyk6bih0LHIpKXx8cyk7cmV0dXJuIG8+MyYmcyYmT2JqZWN0LmRlZmluZVByb3BlcnR5KHQscixzKSxzfSxzPXRoaXMmJnRoaXMuX19wYXJhbXx8ZnVuY3Rpb24oZSx0KXtyZXR1cm4gZnVuY3Rpb24ocixpKXt0KHIsaSxlKX19O09iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LCJfX2VzTW9kdWxlIix7dmFsdWU6ITB9KSx0LlRleHRSZW5kZXJMYXllcj12b2lkIDA7dmFyIGE9cigzNzAwKSxjPXIoMTU0NiksbD1yKDM3MzQpLHU9cig2NDMpLGg9cig1MTEpLGY9cigyNTg1KSxfPXIoNDcyNSksZD1yKDQyNjkpLHA9ZnVuY3Rpb24oZSl7ZnVuY3Rpb24gdCh0LHIsaSxuLG8scyxjLGwpe3ZhciB1PWUuY2FsbCh0aGlzLHQsInRleHQiLHIsbixpLG8scyxjKXx8dGhpcztyZXR1cm4gdS5fY2hhcmFjdGVySm9pbmVyU2VydmljZT1sLHUuX2NoYXJhY3RlcldpZHRoPTAsdS5fY2hhcmFjdGVyRm9udD0iIix1Ll9jaGFyYWN0ZXJPdmVybGFwQ2FjaGU9e30sdS5fd29ya0NlbGw9bmV3IGguQ2VsbERhdGEsdS5fc3RhdGU9bmV3IGEuR3JpZENhY2hlLHV9cmV0dXJuIG4odCxlKSx0LnByb3RvdHlwZS5yZXNpemU9ZnVuY3Rpb24odCl7ZS5wcm90b3R5cGUucmVzaXplLmNhbGwodGhpcyx0KTt2YXIgcj10aGlzLl9nZXRGb250KCExLCExKTt0aGlzLl9jaGFyYWN0ZXJXaWR0aD09PXQuc2NhbGVkQ2hhcldpZHRoJiZ0aGlzLl9jaGFyYWN0ZXJGb250PT09cnx8KHRoaXMuX2NoYXJhY3RlcldpZHRoPXQuc2NhbGVkQ2hhcldpZHRoLHRoaXMuX2NoYXJhY3RlckZvbnQ9cix0aGlzLl9jaGFyYWN0ZXJPdmVybGFwQ2FjaGU9e30pLHRoaXMuX3N0YXRlLmNsZWFyKCksdGhpcy5fc3RhdGUucmVzaXplKHRoaXMuX2J1ZmZlclNlcnZpY2UuY29scyx0aGlzLl9idWZmZXJTZXJ2aWNlLnJvd3MpfSx0LnByb3RvdHlwZS5yZXNldD1mdW5jdGlvbigpe3RoaXMuX3N0YXRlLmNsZWFyKCksdGhpcy5fY2xlYXJBbGwoKX0sdC5wcm90b3R5cGUuX2ZvckVhY2hDZWxsPWZ1bmN0aW9uKGUsdCxyKXtmb3IodmFyIGk9ZTtpPD10O2krKylmb3IodmFyIG49aSt0aGlzLl9idWZmZXJTZXJ2aWNlLmJ1ZmZlci55ZGlzcCxvPXRoaXMuX2J1ZmZlclNlcnZpY2UuYnVmZmVyLmxpbmVzLmdldChuKSxzPXRoaXMuX2NoYXJhY3RlckpvaW5lclNlcnZpY2UuZ2V0Sm9pbmVkQ2hhcmFjdGVycyhuKSxhPTA7YTx0aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHM7YSsrKXtvLmxvYWRDZWxsKGEsdGhpcy5fd29ya0NlbGwpO3ZhciBjPXRoaXMuX3dvcmtDZWxsLGw9ITEsaD1hO2lmKDAhPT1jLmdldFdpZHRoKCkpe2lmKHMubGVuZ3RoPjAmJmE9PT1zWzBdWzBdKXtsPSEwO3ZhciBmPXMuc2hpZnQoKTtjPW5ldyBkLkpvaW5lZENlbGxEYXRhKHRoaXMuX3dvcmtDZWxsLG8udHJhbnNsYXRlVG9TdHJpbmcoITAsZlswXSxmWzFdKSxmWzFdLWZbMF0pLGg9ZlsxXS0xfSFsJiZ0aGlzLl9pc092ZXJsYXBwaW5nKGMpJiZoPG8ubGVuZ3RoLTEmJm8uZ2V0Q29kZVBvaW50KGgrMSk9PT11Lk5VTExfQ0VMTF9DT0RFJiYoYy5jb250ZW50Jj0tMTI1ODI5MTMsYy5jb250ZW50fD0yPDwyMikscihjLGEsaSksYT1ofX19LHQucHJvdG90eXBlLl9kcmF3QmFja2dyb3VuZD1mdW5jdGlvbihlLHQpe3ZhciByPXRoaXMsaT10aGlzLl9jdHgsbj10aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHMsbz0wLHM9MCxhPW51bGw7aS5zYXZlKCksdGhpcy5fZm9yRWFjaENlbGwoZSx0LChmdW5jdGlvbihlLHQsYyl7dmFyIHU9bnVsbDtlLmlzSW52ZXJzZSgpP3U9ZS5pc0ZnRGVmYXVsdCgpP3IuX2NvbG9ycy5mb3JlZ3JvdW5kLmNzczplLmlzRmdSR0IoKT8icmdiKCIrbC5BdHRyaWJ1dGVEYXRhLnRvQ29sb3JSR0IoZS5nZXRGZ0NvbG9yKCkpLmpvaW4oIiwiKSsiKSI6ci5fY29sb3JzLmFuc2lbZS5nZXRGZ0NvbG9yKCldLmNzczplLmlzQmdSR0IoKT91PSJyZ2IoIitsLkF0dHJpYnV0ZURhdGEudG9Db2xvclJHQihlLmdldEJnQ29sb3IoKSkuam9pbigiLCIpKyIpIjplLmlzQmdQYWxldHRlKCkmJih1PXIuX2NvbG9ycy5hbnNpW2UuZ2V0QmdDb2xvcigpXS5jc3MpLG51bGw9PT1hJiYobz10LHM9YyksYyE9PXM/KGkuZmlsbFN0eWxlPWF8fCIiLHIuX2ZpbGxDZWxscyhvLHMsbi1vLDEpLG89dCxzPWMpOmEhPT11JiYoaS5maWxsU3R5bGU9YXx8IiIsci5fZmlsbENlbGxzKG8scyx0LW8sMSksbz10LHM9YyksYT11fSkpLG51bGwhPT1hJiYoaS5maWxsU3R5bGU9YSx0aGlzLl9maWxsQ2VsbHMobyxzLG4tbywxKSksaS5yZXN0b3JlKCl9LHQucHJvdG90eXBlLl9kcmF3Rm9yZWdyb3VuZD1mdW5jdGlvbihlLHQpe3ZhciByPXRoaXM7dGhpcy5fZm9yRWFjaENlbGwoZSx0LChmdW5jdGlvbihlLHQsaSl7aWYoIWUuaXNJbnZpc2libGUoKSYmKHIuX2RyYXdDaGFycyhlLHQsaSksZS5pc1VuZGVybGluZSgpfHxlLmlzU3RyaWtldGhyb3VnaCgpKSl7aWYoci5fY3R4LnNhdmUoKSxlLmlzSW52ZXJzZSgpKWlmKGUuaXNCZ0RlZmF1bHQoKSlyLl9jdHguZmlsbFN0eWxlPXIuX2NvbG9ycy5iYWNrZ3JvdW5kLmNzcztlbHNlIGlmKGUuaXNCZ1JHQigpKXIuX2N0eC5maWxsU3R5bGU9InJnYigiK2wuQXR0cmlidXRlRGF0YS50b0NvbG9yUkdCKGUuZ2V0QmdDb2xvcigpKS5qb2luKCIsIikrIikiO2Vsc2V7dmFyIG49ZS5nZXRCZ0NvbG9yKCk7ci5fb3B0aW9uc1NlcnZpY2Uub3B0aW9ucy5kcmF3Qm9sZFRleHRJbkJyaWdodENvbG9ycyYmZS5pc0JvbGQoKSYmbjw4JiYobis9OCksci5fY3R4LmZpbGxTdHlsZT1yLl9jb2xvcnMuYW5zaVtuXS5jc3N9ZWxzZSBpZihlLmlzRmdEZWZhdWx0KCkpci5fY3R4LmZpbGxTdHlsZT1yLl9jb2xvcnMuZm9yZWdyb3VuZC5jc3M7ZWxzZSBpZihlLmlzRmdSR0IoKSlyLl9jdHguZmlsbFN0eWxlPSJyZ2IoIitsLkF0dHJpYnV0ZURhdGEudG9Db2xvclJHQihlLmdldEZnQ29sb3IoKSkuam9pbigiLCIpKyIpIjtlbHNle3ZhciBvPWUuZ2V0RmdDb2xvcigpO3IuX29wdGlvbnNTZXJ2aWNlLm9wdGlvbnMuZHJhd0JvbGRUZXh0SW5CcmlnaHRDb2xvcnMmJmUuaXNCb2xkKCkmJm88OCYmKG8rPTgpLHIuX2N0eC5maWxsU3R5bGU9ci5fY29sb3JzLmFuc2lbb10uY3NzfWUuaXNTdHJpa2V0aHJvdWdoKCkmJnIuX2ZpbGxNaWRkbGVMaW5lQXRDZWxscyh0LGksZS5nZXRXaWR0aCgpKSxlLmlzVW5kZXJsaW5lKCkmJnIuX2ZpbGxCb3R0b21MaW5lQXRDZWxscyh0LGksZS5nZXRXaWR0aCgpKSxyLl9jdHgucmVzdG9yZSgpfX0pKX0sdC5wcm90b3R5cGUub25HcmlkQ2hhbmdlZD1mdW5jdGlvbihlLHQpezAhPT10aGlzLl9zdGF0ZS5jYWNoZS5sZW5ndGgmJih0aGlzLl9jaGFyQXRsYXMmJnRoaXMuX2NoYXJBdGxhcy5iZWdpbkZyYW1lKCksdGhpcy5fY2xlYXJDZWxscygwLGUsdGhpcy5fYnVmZmVyU2VydmljZS5jb2xzLHQtZSsxKSx0aGlzLl9kcmF3QmFja2dyb3VuZChlLHQpLHRoaXMuX2RyYXdGb3JlZ3JvdW5kKGUsdCkpfSx0LnByb3RvdHlwZS5vbk9wdGlvbnNDaGFuZ2VkPWZ1bmN0aW9uKCl7dGhpcy5fc2V0VHJhbnNwYXJlbmN5KHRoaXMuX29wdGlvbnNTZXJ2aWNlLm9wdGlvbnMuYWxsb3dUcmFuc3BhcmVuY3kpfSx0LnByb3RvdHlwZS5faXNPdmVybGFwcGluZz1mdW5jdGlvbihlKXtpZigxIT09ZS5nZXRXaWR0aCgpKXJldHVybiExO2lmKGUuZ2V0Q29kZSgpPDI1NilyZXR1cm4hMTt2YXIgdD1lLmdldENoYXJzKCk7aWYodGhpcy5fY2hhcmFjdGVyT3ZlcmxhcENhY2hlLmhhc093blByb3BlcnR5KHQpKXJldHVybiB0aGlzLl9jaGFyYWN0ZXJPdmVybGFwQ2FjaGVbdF07dGhpcy5fY3R4LnNhdmUoKSx0aGlzLl9jdHguZm9udD10aGlzLl9jaGFyYWN0ZXJGb250O3ZhciByPU1hdGguZmxvb3IodGhpcy5fY3R4Lm1lYXN1cmVUZXh0KHQpLndpZHRoKT50aGlzLl9jaGFyYWN0ZXJXaWR0aDtyZXR1cm4gdGhpcy5fY3R4LnJlc3RvcmUoKSx0aGlzLl9jaGFyYWN0ZXJPdmVybGFwQ2FjaGVbdF09cixyfSxvKFtzKDUsZi5JQnVmZmVyU2VydmljZSkscyg2LGYuSU9wdGlvbnNTZXJ2aWNlKSxzKDcsXy5JQ2hhcmFjdGVySm9pbmVyU2VydmljZSldLHQpfShjLkJhc2VSZW5kZXJMYXllcik7dC5UZXh0UmVuZGVyTGF5ZXI9cH0sOTYxNjooZSx0KT0+e09iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LCJfX2VzTW9kdWxlIix7dmFsdWU6ITB9KSx0LkJhc2VDaGFyQXRsYXM9dm9pZCAwO3ZhciByPWZ1bmN0aW9uKCl7ZnVuY3Rpb24gZSgpe3RoaXMuX2RpZFdhcm1VcD0hMX1yZXR1cm4gZS5wcm90b3R5cGUuZGlzcG9zZT1mdW5jdGlvbigpe30sZS5wcm90b3R5cGUud2FybVVwPWZ1bmN0aW9uKCl7dGhpcy5fZGlkV2FybVVwfHwodGhpcy5fZG9XYXJtVXAoKSx0aGlzLl9kaWRXYXJtVXA9ITApfSxlLnByb3RvdHlwZS5fZG9XYXJtVXA9ZnVuY3Rpb24oKXt9LGUucHJvdG90eXBlLmNsZWFyPWZ1bmN0aW9uKCl7fSxlLnByb3RvdHlwZS5iZWdpbkZyYW1lPWZ1bmN0aW9uKCl7fSxlfSgpO3QuQmFzZUNoYXJBdGxhcz1yfSwxNDIwOihlLHQscik9PntPYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC5yZW1vdmVUZXJtaW5hbEZyb21DYWNoZT10LmFjcXVpcmVDaGFyQXRsYXM9dm9pZCAwO3ZhciBpPXIoMjA0MCksbj1yKDE5MDYpLG89W107dC5hY3F1aXJlQ2hhckF0bGFzPWZ1bmN0aW9uKGUsdCxyLHMsYSl7Zm9yKHZhciBjPSgwLGkuZ2VuZXJhdGVDb25maWcpKHMsYSxlLHIpLGw9MDtsPG8ubGVuZ3RoO2wrKyl7dmFyIHU9KGg9b1tsXSkub3duZWRCeS5pbmRleE9mKHQpO2lmKHU+PTApe2lmKCgwLGkuY29uZmlnRXF1YWxzKShoLmNvbmZpZyxjKSlyZXR1cm4gaC5hdGxhczsxPT09aC5vd25lZEJ5Lmxlbmd0aD8oaC5hdGxhcy5kaXNwb3NlKCksby5zcGxpY2UobCwxKSk6aC5vd25lZEJ5LnNwbGljZSh1LDEpO2JyZWFrfX1mb3IobD0wO2w8by5sZW5ndGg7bCsrKXt2YXIgaD1vW2xdO2lmKCgwLGkuY29uZmlnRXF1YWxzKShoLmNvbmZpZyxjKSlyZXR1cm4gaC5vd25lZEJ5LnB1c2godCksaC5hdGxhc312YXIgZj17YXRsYXM6bmV3IG4uRHluYW1pY0NoYXJBdGxhcyhkb2N1bWVudCxjKSxjb25maWc6Yyxvd25lZEJ5Olt0XX07cmV0dXJuIG8ucHVzaChmKSxmLmF0bGFzfSx0LnJlbW92ZVRlcm1pbmFsRnJvbUNhY2hlPWZ1bmN0aW9uKGUpe2Zvcih2YXIgdD0wO3Q8by5sZW5ndGg7dCsrKXt2YXIgcj1vW3RdLm93bmVkQnkuaW5kZXhPZihlKTtpZigtMSE9PXIpezE9PT1vW3RdLm93bmVkQnkubGVuZ3RoPyhvW3RdLmF0bGFzLmRpc3Bvc2UoKSxvLnNwbGljZSh0LDEpKTpvW3RdLm93bmVkQnkuc3BsaWNlKHIsMSk7YnJlYWt9fX19LDIwNDA6ZnVuY3Rpb24oZSx0LHIpe3ZhciBpPXRoaXMmJnRoaXMuX19zcHJlYWRBcnJheXx8ZnVuY3Rpb24oZSx0LHIpe2lmKHJ8fDI9PT1hcmd1bWVudHMubGVuZ3RoKWZvcih2YXIgaSxuPTAsbz10Lmxlbmd0aDtuPG87bisrKSFpJiZuIGluIHR8fChpfHwoaT1BcnJheS5wcm90b3R5cGUuc2xpY2UuY2FsbCh0LDAsbikpLGlbbl09dFtuXSk7cmV0dXJuIGUuY29uY2F0KGl8fEFycmF5LnByb3RvdHlwZS5zbGljZS5jYWxsKHQpKX07T2JqZWN0LmRlZmluZVByb3BlcnR5KHQsIl9fZXNNb2R1bGUiLHt2YWx1ZTohMH0pLHQuaXMyNTZDb2xvcj10LmNvbmZpZ0VxdWFscz10LmdlbmVyYXRlQ29uZmlnPXZvaWQgMDt2YXIgbj1yKDY0Myk7dC5nZW5lcmF0ZUNvbmZpZz1mdW5jdGlvbihlLHQscixuKXt2YXIgbz17Zm9yZWdyb3VuZDpuLmZvcmVncm91bmQsYmFja2dyb3VuZDpuLmJhY2tncm91bmQsY3Vyc29yOnZvaWQgMCxjdXJzb3JBY2NlbnQ6dm9pZCAwLHNlbGVjdGlvbjp2b2lkIDAsYW5zaTppKFtdLG4uYW5zaSwhMCl9O3JldHVybntkZXZpY2VQaXhlbFJhdGlvOndpbmRvdy5kZXZpY2VQaXhlbFJhdGlvLHNjYWxlZENoYXJXaWR0aDplLHNjYWxlZENoYXJIZWlnaHQ6dCxmb250RmFtaWx5OnIuZm9udEZhbWlseSxmb250U2l6ZTpyLmZvbnRTaXplLGZvbnRXZWlnaHQ6ci5mb250V2VpZ2h0LGZvbnRXZWlnaHRCb2xkOnIuZm9udFdlaWdodEJvbGQsYWxsb3dUcmFuc3BhcmVuY3k6ci5hbGxvd1RyYW5zcGFyZW5jeSxjb2xvcnM6b319LHQuY29uZmlnRXF1YWxzPWZ1bmN0aW9uKGUsdCl7Zm9yKHZhciByPTA7cjxlLmNvbG9ycy5hbnNpLmxlbmd0aDtyKyspaWYoZS5jb2xvcnMuYW5zaVtyXS5yZ2JhIT09dC5jb2xvcnMuYW5zaVtyXS5yZ2JhKXJldHVybiExO3JldHVybiBlLmRldmljZVBpeGVsUmF0aW89PT10LmRldmljZVBpeGVsUmF0aW8mJmUuZm9udEZhbWlseT09PXQuZm9udEZhbWlseSYmZS5mb250U2l6ZT09PXQuZm9udFNpemUmJmUuZm9udFdlaWdodD09PXQuZm9udFdlaWdodCYmZS5mb250V2VpZ2h0Qm9sZD09PXQuZm9udFdlaWdodEJvbGQmJmUuYWxsb3dUcmFuc3BhcmVuY3k9PT10LmFsbG93VHJhbnNwYXJlbmN5JiZlLnNjYWxlZENoYXJXaWR0aD09PXQuc2NhbGVkQ2hhcldpZHRoJiZlLnNjYWxlZENoYXJIZWlnaHQ9PT10LnNjYWxlZENoYXJIZWlnaHQmJmUuY29sb3JzLmZvcmVncm91bmQ9PT10LmNvbG9ycy5mb3JlZ3JvdW5kJiZlLmNvbG9ycy5iYWNrZ3JvdW5kPT09dC5jb2xvcnMuYmFja2dyb3VuZH0sdC5pczI1NkNvbG9yPWZ1bmN0aW9uKGUpe3JldHVybiBlPG4uREVGQVVMVF9DT0xPUn19LDg4MDM6KGUsdCxyKT0+e09iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LCJfX2VzTW9kdWxlIix7dmFsdWU6ITB9KSx0LkNIQVJfQVRMQVNfQ0VMTF9TUEFDSU5HPXQuVEVYVF9CQVNFTElORT10LkRJTV9PUEFDSVRZPXQuSU5WRVJURURfREVGQVVMVF9DT0xPUj12b2lkIDA7dmFyIGk9cig2MTE0KTt0LklOVkVSVEVEX0RFRkFVTFRfQ09MT1I9MjU3LHQuRElNX09QQUNJVFk9LjUsdC5URVhUX0JBU0VMSU5FPWkuaXNGaXJlZm94PyJib3R0b20iOiJpZGVvZ3JhcGhpYyIsdC5DSEFSX0FUTEFTX0NFTExfU1BBQ0lORz0xfSwxOTA2OmZ1bmN0aW9uKGUsdCxyKXt2YXIgaSxuPXRoaXMmJnRoaXMuX19leHRlbmRzfHwoaT1mdW5jdGlvbihlLHQpe3JldHVybiBpPU9iamVjdC5zZXRQcm90b3R5cGVPZnx8e19fcHJvdG9fXzpbXX1pbnN0YW5jZW9mIEFycmF5JiZmdW5jdGlvbihlLHQpe2UuX19wcm90b19fPXR9fHxmdW5jdGlvbihlLHQpe2Zvcih2YXIgciBpbiB0KU9iamVjdC5wcm90b3R5cGUuaGFzT3duUHJvcGVydHkuY2FsbCh0LHIpJiYoZVtyXT10W3JdKX0saShlLHQpfSxmdW5jdGlvbihlLHQpe2lmKCJmdW5jdGlvbiIhPXR5cGVvZiB0JiZudWxsIT09dCl0aHJvdyBuZXcgVHlwZUVycm9yKCJDbGFzcyBleHRlbmRzIHZhbHVlICIrU3RyaW5nKHQpKyIgaXMgbm90IGEgY29uc3RydWN0b3Igb3IgbnVsbCIpO2Z1bmN0aW9uIHIoKXt0aGlzLmNvbnN0cnVjdG9yPWV9aShlLHQpLGUucHJvdG90eXBlPW51bGw9PT10P09iamVjdC5jcmVhdGUodCk6KHIucHJvdG90eXBlPXQucHJvdG90eXBlLG5ldyByKX0pO09iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LCJfX2VzTW9kdWxlIix7dmFsdWU6ITB9KSx0Lk5vbmVDaGFyQXRsYXM9dC5EeW5hbWljQ2hhckF0bGFzPXQuZ2V0R2x5cGhDYWNoZUtleT12b2lkIDA7dmFyIG89cig4ODAzKSxzPXIoOTYxNiksYT1yKDU2ODApLGM9cig3MDAxKSxsPXIoNjExNCksdT1yKDE3NTIpLGg9cig0Nzc0KSxmPTEwMjQsXz0xMDI0LGQ9e2NzczoicmdiYSgwLCAwLCAwLCAwKSIscmdiYTowfTtmdW5jdGlvbiBwKGUpe3JldHVybiBlLmNvZGU8PDIxfGUuYmc8PDEyfGUuZmc8PDN8KGUuYm9sZD8wOjQpKyhlLmRpbT8wOjIpKyhlLml0YWxpYz8wOjEpfXQuZ2V0R2x5cGhDYWNoZUtleT1wO3ZhciB2PWZ1bmN0aW9uKGUpe2Z1bmN0aW9uIHQodCxyKXt2YXIgaT1lLmNhbGwodGhpcyl8fHRoaXM7aS5fY29uZmlnPXIsaS5fZHJhd1RvQ2FjaGVDb3VudD0wLGkuX2dseXBoc1dhaXRpbmdPbkJpdG1hcD1bXSxpLl9iaXRtYXBDb21taXRUaW1lb3V0PW51bGwsaS5fYml0bWFwPW51bGwsaS5fY2FjaGVDYW52YXM9dC5jcmVhdGVFbGVtZW50KCJjYW52YXMiKSxpLl9jYWNoZUNhbnZhcy53aWR0aD1mLGkuX2NhY2hlQ2FudmFzLmhlaWdodD1fLGkuX2NhY2hlQ3R4PSgwLHUudGhyb3dJZkZhbHN5KShpLl9jYWNoZUNhbnZhcy5nZXRDb250ZXh0KCIyZCIse2FscGhhOiEwfSkpO3ZhciBuPXQuY3JlYXRlRWxlbWVudCgiY2FudmFzIik7bi53aWR0aD1pLl9jb25maWcuc2NhbGVkQ2hhcldpZHRoLG4uaGVpZ2h0PWkuX2NvbmZpZy5zY2FsZWRDaGFySGVpZ2h0LGkuX3RtcEN0eD0oMCx1LnRocm93SWZGYWxzeSkobi5nZXRDb250ZXh0KCIyZCIse2FscGhhOmkuX2NvbmZpZy5hbGxvd1RyYW5zcGFyZW5jeX0pKSxpLl93aWR0aD1NYXRoLmZsb29yKGYvaS5fY29uZmlnLnNjYWxlZENoYXJXaWR0aCksaS5faGVpZ2h0PU1hdGguZmxvb3IoXy9pLl9jb25maWcuc2NhbGVkQ2hhckhlaWdodCk7dmFyIG89aS5fd2lkdGgqaS5faGVpZ2h0O3JldHVybiBpLl9jYWNoZU1hcD1uZXcgYy5MUlVNYXAobyksaS5fY2FjaGVNYXAucHJlYWxsb2MobyksaX1yZXR1cm4gbih0LGUpLHQucHJvdG90eXBlLmRpc3Bvc2U9ZnVuY3Rpb24oKXtudWxsIT09dGhpcy5fYml0bWFwQ29tbWl0VGltZW91dCYmKHdpbmRvdy5jbGVhclRpbWVvdXQodGhpcy5fYml0bWFwQ29tbWl0VGltZW91dCksdGhpcy5fYml0bWFwQ29tbWl0VGltZW91dD1udWxsKX0sdC5wcm90b3R5cGUuYmVnaW5GcmFtZT1mdW5jdGlvbigpe3RoaXMuX2RyYXdUb0NhY2hlQ291bnQ9MH0sdC5wcm90b3R5cGUuY2xlYXI9ZnVuY3Rpb24oKXtpZih0aGlzLl9jYWNoZU1hcC5zaXplPjApe3ZhciBlPXRoaXMuX3dpZHRoKnRoaXMuX2hlaWdodDt0aGlzLl9jYWNoZU1hcD1uZXcgYy5MUlVNYXAoZSksdGhpcy5fY2FjaGVNYXAucHJlYWxsb2MoZSl9dGhpcy5fY2FjaGVDdHguY2xlYXJSZWN0KDAsMCxmLF8pLHRoaXMuX3RtcEN0eC5jbGVhclJlY3QoMCwwLHRoaXMuX2NvbmZpZy5zY2FsZWRDaGFyV2lkdGgsdGhpcy5fY29uZmlnLnNjYWxlZENoYXJIZWlnaHQpfSx0LnByb3RvdHlwZS5kcmF3PWZ1bmN0aW9uKGUsdCxyLGkpe2lmKDMyPT09dC5jb2RlKXJldHVybiEwO2lmKCF0aGlzLl9jYW5DYWNoZSh0KSlyZXR1cm4hMTt2YXIgbj1wKHQpLG89dGhpcy5fY2FjaGVNYXAuZ2V0KG4pO2lmKG51bGwhPW8pcmV0dXJuIHRoaXMuX2RyYXdGcm9tQ2FjaGUoZSxvLHIsaSksITA7aWYodGhpcy5fZHJhd1RvQ2FjaGVDb3VudDwxMDApe3ZhciBzO3M9dGhpcy5fY2FjaGVNYXAuc2l6ZTx0aGlzLl9jYWNoZU1hcC5jYXBhY2l0eT90aGlzLl9jYWNoZU1hcC5zaXplOnRoaXMuX2NhY2hlTWFwLnBlZWsoKS5pbmRleDt2YXIgYT10aGlzLl9kcmF3VG9DYWNoZSh0LHMpO3JldHVybiB0aGlzLl9jYWNoZU1hcC5zZXQobixhKSx0aGlzLl9kcmF3RnJvbUNhY2hlKGUsYSxyLGkpLCEwfXJldHVybiExfSx0LnByb3RvdHlwZS5fY2FuQ2FjaGU9ZnVuY3Rpb24oZSl7cmV0dXJuIGUuY29kZTwyNTZ9LHQucHJvdG90eXBlLl90b0Nvb3JkaW5hdGVYPWZ1bmN0aW9uKGUpe3JldHVybiBlJXRoaXMuX3dpZHRoKnRoaXMuX2NvbmZpZy5zY2FsZWRDaGFyV2lkdGh9LHQucHJvdG90eXBlLl90b0Nvb3JkaW5hdGVZPWZ1bmN0aW9uKGUpe3JldHVybiBNYXRoLmZsb29yKGUvdGhpcy5fd2lkdGgpKnRoaXMuX2NvbmZpZy5zY2FsZWRDaGFySGVpZ2h0fSx0LnByb3RvdHlwZS5fZHJhd0Zyb21DYWNoZT1mdW5jdGlvbihlLHQscixpKXtpZighdC5pc0VtcHR5KXt2YXIgbj10aGlzLl90b0Nvb3JkaW5hdGVYKHQuaW5kZXgpLG89dGhpcy5fdG9Db29yZGluYXRlWSh0LmluZGV4KTtlLmRyYXdJbWFnZSh0LmluQml0bWFwP3RoaXMuX2JpdG1hcDp0aGlzLl9jYWNoZUNhbnZhcyxuLG8sdGhpcy5fY29uZmlnLnNjYWxlZENoYXJXaWR0aCx0aGlzLl9jb25maWcuc2NhbGVkQ2hhckhlaWdodCxyLGksdGhpcy5fY29uZmlnLnNjYWxlZENoYXJXaWR0aCx0aGlzLl9jb25maWcuc2NhbGVkQ2hhckhlaWdodCl9fSx0LnByb3RvdHlwZS5fZ2V0Q29sb3JGcm9tQW5zaUluZGV4PWZ1bmN0aW9uKGUpe3JldHVybiBlPHRoaXMuX2NvbmZpZy5jb2xvcnMuYW5zaS5sZW5ndGg/dGhpcy5fY29uZmlnLmNvbG9ycy5hbnNpW2VdOmEuREVGQVVMVF9BTlNJX0NPTE9SU1tlXX0sdC5wcm90b3R5cGUuX2dldEJhY2tncm91bmRDb2xvcj1mdW5jdGlvbihlKXtyZXR1cm4gdGhpcy5fY29uZmlnLmFsbG93VHJhbnNwYXJlbmN5P2Q6ZS5iZz09PW8uSU5WRVJURURfREVGQVVMVF9DT0xPUj90aGlzLl9jb25maWcuY29sb3JzLmZvcmVncm91bmQ6ZS5iZzwyNTY/dGhpcy5fZ2V0Q29sb3JGcm9tQW5zaUluZGV4KGUuYmcpOnRoaXMuX2NvbmZpZy5jb2xvcnMuYmFja2dyb3VuZH0sdC5wcm90b3R5cGUuX2dldEZvcmVncm91bmRDb2xvcj1mdW5jdGlvbihlKXtyZXR1cm4gZS5mZz09PW8uSU5WRVJURURfREVGQVVMVF9DT0xPUj9oLmNvbG9yLm9wYXF1ZSh0aGlzLl9jb25maWcuY29sb3JzLmJhY2tncm91bmQpOmUuZmc8MjU2P3RoaXMuX2dldENvbG9yRnJvbUFuc2lJbmRleChlLmZnKTp0aGlzLl9jb25maWcuY29sb3JzLmZvcmVncm91bmR9LHQucHJvdG90eXBlLl9kcmF3VG9DYWNoZT1mdW5jdGlvbihlLHQpe3RoaXMuX2RyYXdUb0NhY2hlQ291bnQrKyx0aGlzLl90bXBDdHguc2F2ZSgpO3ZhciByPXRoaXMuX2dldEJhY2tncm91bmRDb2xvcihlKTt0aGlzLl90bXBDdHguZ2xvYmFsQ29tcG9zaXRlT3BlcmF0aW9uPSJjb3B5Iix0aGlzLl90bXBDdHguZmlsbFN0eWxlPXIuY3NzLHRoaXMuX3RtcEN0eC5maWxsUmVjdCgwLDAsdGhpcy5fY29uZmlnLnNjYWxlZENoYXJXaWR0aCx0aGlzLl9jb25maWcuc2NhbGVkQ2hhckhlaWdodCksdGhpcy5fdG1wQ3R4Lmdsb2JhbENvbXBvc2l0ZU9wZXJhdGlvbj0ic291cmNlLW92ZXIiO3ZhciBpPWUuYm9sZD90aGlzLl9jb25maWcuZm9udFdlaWdodEJvbGQ6dGhpcy5fY29uZmlnLmZvbnRXZWlnaHQsbj1lLml0YWxpYz8iaXRhbGljIjoiIjt0aGlzLl90bXBDdHguZm9udD1uKyIgIitpKyIgIit0aGlzLl9jb25maWcuZm9udFNpemUqdGhpcy5fY29uZmlnLmRldmljZVBpeGVsUmF0aW8rInB4ICIrdGhpcy5fY29uZmlnLmZvbnRGYW1pbHksdGhpcy5fdG1wQ3R4LnRleHRCYXNlbGluZT1vLlRFWFRfQkFTRUxJTkUsdGhpcy5fdG1wQ3R4LmZpbGxTdHlsZT10aGlzLl9nZXRGb3JlZ3JvdW5kQ29sb3IoZSkuY3NzLGUuZGltJiYodGhpcy5fdG1wQ3R4Lmdsb2JhbEFscGhhPW8uRElNX09QQUNJVFkpLHRoaXMuX3RtcEN0eC5maWxsVGV4dChlLmNoYXJzLDAsdGhpcy5fY29uZmlnLnNjYWxlZENoYXJIZWlnaHQpO3ZhciBzPXRoaXMuX3RtcEN0eC5nZXRJbWFnZURhdGEoMCwwLHRoaXMuX2NvbmZpZy5zY2FsZWRDaGFyV2lkdGgsdGhpcy5fY29uZmlnLnNjYWxlZENoYXJIZWlnaHQpLGE9ITE7aWYodGhpcy5fY29uZmlnLmFsbG93VHJhbnNwYXJlbmN5fHwoYT15KHMscikpLGEmJiJfIj09PWUuY2hhcnMmJiF0aGlzLl9jb25maWcuYWxsb3dUcmFuc3BhcmVuY3kpZm9yKHZhciBjPTE7Yzw9NSYmKHRoaXMuX3RtcEN0eC5maWxsVGV4dChlLmNoYXJzLDAsdGhpcy5fY29uZmlnLnNjYWxlZENoYXJIZWlnaHQtYyksYT15KHM9dGhpcy5fdG1wQ3R4LmdldEltYWdlRGF0YSgwLDAsdGhpcy5fY29uZmlnLnNjYWxlZENoYXJXaWR0aCx0aGlzLl9jb25maWcuc2NhbGVkQ2hhckhlaWdodCkscikpO2MrKyk7dGhpcy5fdG1wQ3R4LnJlc3RvcmUoKTt2YXIgbD10aGlzLl90b0Nvb3JkaW5hdGVYKHQpLHU9dGhpcy5fdG9Db29yZGluYXRlWSh0KTt0aGlzLl9jYWNoZUN0eC5wdXRJbWFnZURhdGEocyxsLHUpO3ZhciBoPXtpbmRleDp0LGlzRW1wdHk6YSxpbkJpdG1hcDohMX07cmV0dXJuIHRoaXMuX2FkZEdseXBoVG9CaXRtYXAoaCksaH0sdC5wcm90b3R5cGUuX2FkZEdseXBoVG9CaXRtYXA9ZnVuY3Rpb24oZSl7dmFyIHQ9dGhpczshKCJjcmVhdGVJbWFnZUJpdG1hcCJpbiB3aW5kb3cpfHxsLmlzRmlyZWZveHx8bC5pc1NhZmFyaXx8KHRoaXMuX2dseXBoc1dhaXRpbmdPbkJpdG1hcC5wdXNoKGUpLG51bGw9PT10aGlzLl9iaXRtYXBDb21taXRUaW1lb3V0JiYodGhpcy5fYml0bWFwQ29tbWl0VGltZW91dD13aW5kb3cuc2V0VGltZW91dCgoZnVuY3Rpb24oKXtyZXR1cm4gdC5fZ2VuZXJhdGVCaXRtYXAoKX0pLDEwMCkpKX0sdC5wcm90b3R5cGUuX2dlbmVyYXRlQml0bWFwPWZ1bmN0aW9uKCl7dmFyIGU9dGhpcyx0PXRoaXMuX2dseXBoc1dhaXRpbmdPbkJpdG1hcDt0aGlzLl9nbHlwaHNXYWl0aW5nT25CaXRtYXA9W10sd2luZG93LmNyZWF0ZUltYWdlQml0bWFwKHRoaXMuX2NhY2hlQ2FudmFzKS50aGVuKChmdW5jdGlvbihyKXtlLl9iaXRtYXA9cjtmb3IodmFyIGk9MDtpPHQubGVuZ3RoO2krKyl0W2ldLmluQml0bWFwPSEwfSkpLHRoaXMuX2JpdG1hcENvbW1pdFRpbWVvdXQ9bnVsbH0sdH0ocy5CYXNlQ2hhckF0bGFzKTt0LkR5bmFtaWNDaGFyQXRsYXM9djt2YXIgZz1mdW5jdGlvbihlKXtmdW5jdGlvbiB0KHQscil7cmV0dXJuIGUuY2FsbCh0aGlzKXx8dGhpc31yZXR1cm4gbih0LGUpLHQucHJvdG90eXBlLmRyYXc9ZnVuY3Rpb24oZSx0LHIsaSl7cmV0dXJuITF9LHR9KHMuQmFzZUNoYXJBdGxhcyk7ZnVuY3Rpb24geShlLHQpe2Zvcih2YXIgcj0hMCxpPXQucmdiYT4+PjI0LG49dC5yZ2JhPj4+MTYmMjU1LG89dC5yZ2JhPj4+OCYyNTUscz0wO3M8ZS5kYXRhLmxlbmd0aDtzKz00KWUuZGF0YVtzXT09PWkmJmUuZGF0YVtzKzFdPT09biYmZS5kYXRhW3MrMl09PT1vP2UuZGF0YVtzKzNdPTA6cj0hMTtyZXR1cm4gcn10Lk5vbmVDaGFyQXRsYXM9Z30sNzAwMTooZSx0KT0+e09iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LCJfX2VzTW9kdWxlIix7dmFsdWU6ITB9KSx0LkxSVU1hcD12b2lkIDA7dmFyIHI9ZnVuY3Rpb24oKXtmdW5jdGlvbiBlKGUpe3RoaXMuY2FwYWNpdHk9ZSx0aGlzLl9tYXA9e30sdGhpcy5faGVhZD1udWxsLHRoaXMuX3RhaWw9bnVsbCx0aGlzLl9ub2RlUG9vbD1bXSx0aGlzLnNpemU9MH1yZXR1cm4gZS5wcm90b3R5cGUuX3VubGlua05vZGU9ZnVuY3Rpb24oZSl7dmFyIHQ9ZS5wcmV2LHI9ZS5uZXh0O2U9PT10aGlzLl9oZWFkJiYodGhpcy5faGVhZD1yKSxlPT09dGhpcy5fdGFpbCYmKHRoaXMuX3RhaWw9dCksbnVsbCE9PXQmJih0Lm5leHQ9ciksbnVsbCE9PXImJihyLnByZXY9dCl9LGUucHJvdG90eXBlLl9hcHBlbmROb2RlPWZ1bmN0aW9uKGUpe3ZhciB0PXRoaXMuX3RhaWw7bnVsbCE9PXQmJih0Lm5leHQ9ZSksZS5wcmV2PXQsZS5uZXh0PW51bGwsdGhpcy5fdGFpbD1lLG51bGw9PT10aGlzLl9oZWFkJiYodGhpcy5faGVhZD1lKX0sZS5wcm90b3R5cGUucHJlYWxsb2M9ZnVuY3Rpb24oZSl7Zm9yKHZhciB0PXRoaXMuX25vZGVQb29sLHI9MDtyPGU7cisrKXQucHVzaCh7cHJldjpudWxsLG5leHQ6bnVsbCxrZXk6bnVsbCx2YWx1ZTpudWxsfSl9LGUucHJvdG90eXBlLmdldD1mdW5jdGlvbihlKXt2YXIgdD10aGlzLl9tYXBbZV07cmV0dXJuIHZvaWQgMCE9PXQ/KHRoaXMuX3VubGlua05vZGUodCksdGhpcy5fYXBwZW5kTm9kZSh0KSx0LnZhbHVlKTpudWxsfSxlLnByb3RvdHlwZS5wZWVrVmFsdWU9ZnVuY3Rpb24oZSl7dmFyIHQ9dGhpcy5fbWFwW2VdO3JldHVybiB2b2lkIDAhPT10P3QudmFsdWU6bnVsbH0sZS5wcm90b3R5cGUucGVlaz1mdW5jdGlvbigpe3ZhciBlPXRoaXMuX2hlYWQ7cmV0dXJuIG51bGw9PT1lP251bGw6ZS52YWx1ZX0sZS5wcm90b3R5cGUuc2V0PWZ1bmN0aW9uKGUsdCl7dmFyIHI9dGhpcy5fbWFwW2VdO2lmKHZvaWQgMCE9PXIpcj10aGlzLl9tYXBbZV0sdGhpcy5fdW5saW5rTm9kZShyKSxyLnZhbHVlPXQ7ZWxzZSBpZih0aGlzLnNpemU+PXRoaXMuY2FwYWNpdHkpcj10aGlzLl9oZWFkLHRoaXMuX3VubGlua05vZGUociksZGVsZXRlIHRoaXMuX21hcFtyLmtleV0sci5rZXk9ZSxyLnZhbHVlPXQsdGhpcy5fbWFwW2VdPXI7ZWxzZXt2YXIgaT10aGlzLl9ub2RlUG9vbDtpLmxlbmd0aD4wPygocj1pLnBvcCgpKS5rZXk9ZSxyLnZhbHVlPXQpOnI9e3ByZXY6bnVsbCxuZXh0Om51bGwsa2V5OmUsdmFsdWU6dH0sdGhpcy5fbWFwW2VdPXIsdGhpcy5zaXplKyt9dGhpcy5fYXBwZW5kTm9kZShyKX0sZX0oKTt0LkxSVU1hcD1yfSwxMjk2OmZ1bmN0aW9uKGUsdCxyKXt2YXIgaSxuPXRoaXMmJnRoaXMuX19leHRlbmRzfHwoaT1mdW5jdGlvbihlLHQpe3JldHVybiBpPU9iamVjdC5zZXRQcm90b3R5cGVPZnx8e19fcHJvdG9fXzpbXX1pbnN0YW5jZW9mIEFycmF5JiZmdW5jdGlvbihlLHQpe2UuX19wcm90b19fPXR9fHxmdW5jdGlvbihlLHQpe2Zvcih2YXIgciBpbiB0KU9iamVjdC5wcm90b3R5cGUuaGFzT3duUHJvcGVydHkuY2FsbCh0LHIpJiYoZVtyXT10W3JdKX0saShlLHQpfSxmdW5jdGlvbihlLHQpe2lmKCJmdW5jdGlvbiIhPXR5cGVvZiB0JiZudWxsIT09dCl0aHJvdyBuZXcgVHlwZUVycm9yKCJDbGFzcyBleHRlbmRzIHZhbHVlICIrU3RyaW5nKHQpKyIgaXMgbm90IGEgY29uc3RydWN0b3Igb3IgbnVsbCIpO2Z1bmN0aW9uIHIoKXt0aGlzLmNvbnN0cnVjdG9yPWV9aShlLHQpLGUucHJvdG90eXBlPW51bGw9PT10P09iamVjdC5jcmVhdGUodCk6KHIucHJvdG90eXBlPXQucHJvdG90eXBlLG5ldyByKX0pLG89dGhpcyYmdGhpcy5fX2RlY29yYXRlfHxmdW5jdGlvbihlLHQscixpKXt2YXIgbixvPWFyZ3VtZW50cy5sZW5ndGgscz1vPDM/dDpudWxsPT09aT9pPU9iamVjdC5nZXRPd25Qcm9wZXJ0eURlc2NyaXB0b3IodCxyKTppO2lmKCJvYmplY3QiPT10eXBlb2YgUmVmbGVjdCYmImZ1bmN0aW9uIj09dHlwZW9mIFJlZmxlY3QuZGVjb3JhdGUpcz1SZWZsZWN0LmRlY29yYXRlKGUsdCxyLGkpO2Vsc2UgZm9yKHZhciBhPWUubGVuZ3RoLTE7YT49MDthLS0pKG49ZVthXSkmJihzPShvPDM/bihzKTpvPjM/bih0LHIscyk6bih0LHIpKXx8cyk7cmV0dXJuIG8+MyYmcyYmT2JqZWN0LmRlZmluZVByb3BlcnR5KHQscixzKSxzfSxzPXRoaXMmJnRoaXMuX19wYXJhbXx8ZnVuY3Rpb24oZSx0KXtyZXR1cm4gZnVuY3Rpb24ocixpKXt0KHIsaSxlKX19O09iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LCJfX2VzTW9kdWxlIix7dmFsdWU6ITB9KSx0LkRvbVJlbmRlcmVyPXZvaWQgMDt2YXIgYT1yKDM3ODcpLGM9cig4ODAzKSxsPXIoODQ0KSx1PXIoNDcyNSksaD1yKDI1ODUpLGY9cig4NDYwKSxfPXIoNDc3NCksZD1yKDk2MzEpLHA9Inh0ZXJtLWRvbS1yZW5kZXJlci1vd25lci0iLHY9Inh0ZXJtLWZnLSIsZz0ieHRlcm0tYmctIix5PSJ4dGVybS1mb2N1cyIsbT0xLGI9ZnVuY3Rpb24oZSl7ZnVuY3Rpb24gdCh0LHIsaSxuLG8scyxjLGwsdSxoKXt2YXIgZj1lLmNhbGwodGhpcyl8fHRoaXM7cmV0dXJuIGYuX2NvbG9ycz10LGYuX2VsZW1lbnQ9cixmLl9zY3JlZW5FbGVtZW50PWksZi5fdmlld3BvcnRFbGVtZW50PW4sZi5fbGlua2lmaWVyPW8sZi5fbGlua2lmaWVyMj1zLGYuX2NoYXJTaXplU2VydmljZT1sLGYuX29wdGlvbnNTZXJ2aWNlPXUsZi5fYnVmZmVyU2VydmljZT1oLGYuX3Rlcm1pbmFsQ2xhc3M9bSsrLGYuX3Jvd0VsZW1lbnRzPVtdLGYuX3Jvd0NvbnRhaW5lcj1kb2N1bWVudC5jcmVhdGVFbGVtZW50KCJkaXYiKSxmLl9yb3dDb250YWluZXIuY2xhc3NMaXN0LmFkZCgieHRlcm0tcm93cyIpLGYuX3Jvd0NvbnRhaW5lci5zdHlsZS5saW5lSGVpZ2h0PSJub3JtYWwiLGYuX3Jvd0NvbnRhaW5lci5zZXRBdHRyaWJ1dGUoImFyaWEtaGlkZGVuIiwidHJ1ZSIpLGYuX3JlZnJlc2hSb3dFbGVtZW50cyhmLl9idWZmZXJTZXJ2aWNlLmNvbHMsZi5fYnVmZmVyU2VydmljZS5yb3dzKSxmLl9zZWxlY3Rpb25Db250YWluZXI9ZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgiZGl2IiksZi5fc2VsZWN0aW9uQ29udGFpbmVyLmNsYXNzTGlzdC5hZGQoInh0ZXJtLXNlbGVjdGlvbiIpLGYuX3NlbGVjdGlvbkNvbnRhaW5lci5zZXRBdHRyaWJ1dGUoImFyaWEtaGlkZGVuIiwidHJ1ZSIpLGYuZGltZW5zaW9ucz17c2NhbGVkQ2hhcldpZHRoOjAsc2NhbGVkQ2hhckhlaWdodDowLHNjYWxlZENlbGxXaWR0aDowLHNjYWxlZENlbGxIZWlnaHQ6MCxzY2FsZWRDaGFyTGVmdDowLHNjYWxlZENoYXJUb3A6MCxzY2FsZWRDYW52YXNXaWR0aDowLHNjYWxlZENhbnZhc0hlaWdodDowLGNhbnZhc1dpZHRoOjAsY2FudmFzSGVpZ2h0OjAsYWN0dWFsQ2VsbFdpZHRoOjAsYWN0dWFsQ2VsbEhlaWdodDowfSxmLl91cGRhdGVEaW1lbnNpb25zKCksZi5faW5qZWN0Q3NzKCksZi5fcm93RmFjdG9yeT1jLmNyZWF0ZUluc3RhbmNlKGEuRG9tUmVuZGVyZXJSb3dGYWN0b3J5LGRvY3VtZW50LGYuX2NvbG9ycyksZi5fZWxlbWVudC5jbGFzc0xpc3QuYWRkKHArZi5fdGVybWluYWxDbGFzcyksZi5fc2NyZWVuRWxlbWVudC5hcHBlbmRDaGlsZChmLl9yb3dDb250YWluZXIpLGYuX3NjcmVlbkVsZW1lbnQuYXBwZW5kQ2hpbGQoZi5fc2VsZWN0aW9uQ29udGFpbmVyKSxmLl9saW5raWZpZXIub25TaG93TGlua1VuZGVybGluZSgoZnVuY3Rpb24oZSl7cmV0dXJuIGYuX29uTGlua0hvdmVyKGUpfSkpLGYuX2xpbmtpZmllci5vbkhpZGVMaW5rVW5kZXJsaW5lKChmdW5jdGlvbihlKXtyZXR1cm4gZi5fb25MaW5rTGVhdmUoZSl9KSksZi5fbGlua2lmaWVyMi5vblNob3dMaW5rVW5kZXJsaW5lKChmdW5jdGlvbihlKXtyZXR1cm4gZi5fb25MaW5rSG92ZXIoZSl9KSksZi5fbGlua2lmaWVyMi5vbkhpZGVMaW5rVW5kZXJsaW5lKChmdW5jdGlvbihlKXtyZXR1cm4gZi5fb25MaW5rTGVhdmUoZSl9KSksZn1yZXR1cm4gbih0LGUpLE9iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LnByb3RvdHlwZSwib25SZXF1ZXN0UmVkcmF3Iix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuKG5ldyBmLkV2ZW50RW1pdHRlcikuZXZlbnR9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksdC5wcm90b3R5cGUuZGlzcG9zZT1mdW5jdGlvbigpe3RoaXMuX2VsZW1lbnQuY2xhc3NMaXN0LnJlbW92ZShwK3RoaXMuX3Rlcm1pbmFsQ2xhc3MpLCgwLGQucmVtb3ZlRWxlbWVudEZyb21QYXJlbnQpKHRoaXMuX3Jvd0NvbnRhaW5lcix0aGlzLl9zZWxlY3Rpb25Db250YWluZXIsdGhpcy5fdGhlbWVTdHlsZUVsZW1lbnQsdGhpcy5fZGltZW5zaW9uc1N0eWxlRWxlbWVudCksZS5wcm90b3R5cGUuZGlzcG9zZS5jYWxsKHRoaXMpfSx0LnByb3RvdHlwZS5fdXBkYXRlRGltZW5zaW9ucz1mdW5jdGlvbigpe3RoaXMuZGltZW5zaW9ucy5zY2FsZWRDaGFyV2lkdGg9dGhpcy5fY2hhclNpemVTZXJ2aWNlLndpZHRoKndpbmRvdy5kZXZpY2VQaXhlbFJhdGlvLHRoaXMuZGltZW5zaW9ucy5zY2FsZWRDaGFySGVpZ2h0PU1hdGguY2VpbCh0aGlzLl9jaGFyU2l6ZVNlcnZpY2UuaGVpZ2h0KndpbmRvdy5kZXZpY2VQaXhlbFJhdGlvKSx0aGlzLmRpbWVuc2lvbnMuc2NhbGVkQ2VsbFdpZHRoPXRoaXMuZGltZW5zaW9ucy5zY2FsZWRDaGFyV2lkdGgrTWF0aC5yb3VuZCh0aGlzLl9vcHRpb25zU2VydmljZS5vcHRpb25zLmxldHRlclNwYWNpbmcpLHRoaXMuZGltZW5zaW9ucy5zY2FsZWRDZWxsSGVpZ2h0PU1hdGguZmxvb3IodGhpcy5kaW1lbnNpb25zLnNjYWxlZENoYXJIZWlnaHQqdGhpcy5fb3B0aW9uc1NlcnZpY2Uub3B0aW9ucy5saW5lSGVpZ2h0KSx0aGlzLmRpbWVuc2lvbnMuc2NhbGVkQ2hhckxlZnQ9MCx0aGlzLmRpbWVuc2lvbnMuc2NhbGVkQ2hhclRvcD0wLHRoaXMuZGltZW5zaW9ucy5zY2FsZWRDYW52YXNXaWR0aD10aGlzLmRpbWVuc2lvbnMuc2NhbGVkQ2VsbFdpZHRoKnRoaXMuX2J1ZmZlclNlcnZpY2UuY29scyx0aGlzLmRpbWVuc2lvbnMuc2NhbGVkQ2FudmFzSGVpZ2h0PXRoaXMuZGltZW5zaW9ucy5zY2FsZWRDZWxsSGVpZ2h0KnRoaXMuX2J1ZmZlclNlcnZpY2Uucm93cyx0aGlzLmRpbWVuc2lvbnMuY2FudmFzV2lkdGg9TWF0aC5yb3VuZCh0aGlzLmRpbWVuc2lvbnMuc2NhbGVkQ2FudmFzV2lkdGgvd2luZG93LmRldmljZVBpeGVsUmF0aW8pLHRoaXMuZGltZW5zaW9ucy5jYW52YXNIZWlnaHQ9TWF0aC5yb3VuZCh0aGlzLmRpbWVuc2lvbnMuc2NhbGVkQ2FudmFzSGVpZ2h0L3dpbmRvdy5kZXZpY2VQaXhlbFJhdGlvKSx0aGlzLmRpbWVuc2lvbnMuYWN0dWFsQ2VsbFdpZHRoPXRoaXMuZGltZW5zaW9ucy5jYW52YXNXaWR0aC90aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHMsdGhpcy5kaW1lbnNpb25zLmFjdHVhbENlbGxIZWlnaHQ9dGhpcy5kaW1lbnNpb25zLmNhbnZhc0hlaWdodC90aGlzLl9idWZmZXJTZXJ2aWNlLnJvd3M7Zm9yKHZhciBlPTAsdD10aGlzLl9yb3dFbGVtZW50cztlPHQubGVuZ3RoO2UrKyl7dmFyIHI9dFtlXTtyLnN0eWxlLndpZHRoPXRoaXMuZGltZW5zaW9ucy5jYW52YXNXaWR0aCsicHgiLHIuc3R5bGUuaGVpZ2h0PXRoaXMuZGltZW5zaW9ucy5hY3R1YWxDZWxsSGVpZ2h0KyJweCIsci5zdHlsZS5saW5lSGVpZ2h0PXRoaXMuZGltZW5zaW9ucy5hY3R1YWxDZWxsSGVpZ2h0KyJweCIsci5zdHlsZS5vdmVyZmxvdz0iaGlkZGVuIn10aGlzLl9kaW1lbnNpb25zU3R5bGVFbGVtZW50fHwodGhpcy5fZGltZW5zaW9uc1N0eWxlRWxlbWVudD1kb2N1bWVudC5jcmVhdGVFbGVtZW50KCJzdHlsZSIpLHRoaXMuX3NjcmVlbkVsZW1lbnQuYXBwZW5kQ2hpbGQodGhpcy5fZGltZW5zaW9uc1N0eWxlRWxlbWVudCkpO3ZhciBpPXRoaXMuX3Rlcm1pbmFsU2VsZWN0b3IrIiAueHRlcm0tcm93cyBzcGFuIHsgZGlzcGxheTogaW5saW5lLWJsb2NrOyBoZWlnaHQ6IDEwMCU7IHZlcnRpY2FsLWFsaWduOiB0b3A7IHdpZHRoOiAiK3RoaXMuZGltZW5zaW9ucy5hY3R1YWxDZWxsV2lkdGgrInB4fSI7dGhpcy5fZGltZW5zaW9uc1N0eWxlRWxlbWVudC50ZXh0Q29udGVudD1pLHRoaXMuX3NlbGVjdGlvbkNvbnRhaW5lci5zdHlsZS5oZWlnaHQ9dGhpcy5fdmlld3BvcnRFbGVtZW50LnN0eWxlLmhlaWdodCx0aGlzLl9zY3JlZW5FbGVtZW50LnN0eWxlLndpZHRoPXRoaXMuZGltZW5zaW9ucy5jYW52YXNXaWR0aCsicHgiLHRoaXMuX3NjcmVlbkVsZW1lbnQuc3R5bGUuaGVpZ2h0PXRoaXMuZGltZW5zaW9ucy5jYW52YXNIZWlnaHQrInB4In0sdC5wcm90b3R5cGUuc2V0Q29sb3JzPWZ1bmN0aW9uKGUpe3RoaXMuX2NvbG9ycz1lLHRoaXMuX2luamVjdENzcygpfSx0LnByb3RvdHlwZS5faW5qZWN0Q3NzPWZ1bmN0aW9uKCl7dmFyIGU9dGhpczt0aGlzLl90aGVtZVN0eWxlRWxlbWVudHx8KHRoaXMuX3RoZW1lU3R5bGVFbGVtZW50PWRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoInN0eWxlIiksdGhpcy5fc2NyZWVuRWxlbWVudC5hcHBlbmRDaGlsZCh0aGlzLl90aGVtZVN0eWxlRWxlbWVudCkpO3ZhciB0PXRoaXMuX3Rlcm1pbmFsU2VsZWN0b3IrIiAueHRlcm0tcm93cyB7IGNvbG9yOiAiK3RoaXMuX2NvbG9ycy5mb3JlZ3JvdW5kLmNzcysiOyBmb250LWZhbWlseTogIit0aGlzLl9vcHRpb25zU2VydmljZS5vcHRpb25zLmZvbnRGYW1pbHkrIjsgZm9udC1zaXplOiAiK3RoaXMuX29wdGlvbnNTZXJ2aWNlLm9wdGlvbnMuZm9udFNpemUrInB4O30iO3QrPXRoaXMuX3Rlcm1pbmFsU2VsZWN0b3IrIiBzcGFuOm5vdCguIithLkJPTERfQ0xBU1MrIikgeyBmb250LXdlaWdodDogIit0aGlzLl9vcHRpb25zU2VydmljZS5vcHRpb25zLmZvbnRXZWlnaHQrIjt9Iit0aGlzLl90ZXJtaW5hbFNlbGVjdG9yKyIgc3Bhbi4iK2EuQk9MRF9DTEFTUysiIHsgZm9udC13ZWlnaHQ6ICIrdGhpcy5fb3B0aW9uc1NlcnZpY2Uub3B0aW9ucy5mb250V2VpZ2h0Qm9sZCsiO30iK3RoaXMuX3Rlcm1pbmFsU2VsZWN0b3IrIiBzcGFuLiIrYS5JVEFMSUNfQ0xBU1MrIiB7IGZvbnQtc3R5bGU6IGl0YWxpYzt9Iix0Kz0iQGtleWZyYW1lcyBibGlua19ib3hfc2hhZG93XyIrdGhpcy5fdGVybWluYWxDbGFzcysiIHsgNTAlIHsgIGJveC1zaGFkb3c6IG5vbmU7IH19Iix0Kz0iQGtleWZyYW1lcyBibGlua19ibG9ja18iK3RoaXMuX3Rlcm1pbmFsQ2xhc3MrIiB7IDAlIHsgIGJhY2tncm91bmQtY29sb3I6ICIrdGhpcy5fY29sb3JzLmN1cnNvci5jc3MrIjsgIGNvbG9yOiAiK3RoaXMuX2NvbG9ycy5jdXJzb3JBY2NlbnQuY3NzKyI7IH0gNTAlIHsgIGJhY2tncm91bmQtY29sb3I6ICIrdGhpcy5fY29sb3JzLmN1cnNvckFjY2VudC5jc3MrIjsgIGNvbG9yOiAiK3RoaXMuX2NvbG9ycy5jdXJzb3IuY3NzKyI7IH19Iix0Kz10aGlzLl90ZXJtaW5hbFNlbGVjdG9yKyIgLnh0ZXJtLXJvd3M6bm90KC54dGVybS1mb2N1cykgLiIrYS5DVVJTT1JfQ0xBU1MrIi4iK2EuQ1VSU09SX1NUWUxFX0JMT0NLX0NMQVNTKyIgeyBvdXRsaW5lOiAxcHggc29saWQgIit0aGlzLl9jb2xvcnMuY3Vyc29yLmNzcysiOyBvdXRsaW5lLW9mZnNldDogLTFweDt9Iit0aGlzLl90ZXJtaW5hbFNlbGVjdG9yKyIgLnh0ZXJtLXJvd3MueHRlcm0tZm9jdXMgLiIrYS5DVVJTT1JfQ0xBU1MrIi4iK2EuQ1VSU09SX0JMSU5LX0NMQVNTKyI6bm90KC4iK2EuQ1VSU09SX1NUWUxFX0JMT0NLX0NMQVNTKyIpIHsgYW5pbWF0aW9uOiBibGlua19ib3hfc2hhZG93XyIrdGhpcy5fdGVybWluYWxDbGFzcysiIDFzIHN0ZXAtZW5kIGluZmluaXRlO30iK3RoaXMuX3Rlcm1pbmFsU2VsZWN0b3IrIiAueHRlcm0tcm93cy54dGVybS1mb2N1cyAuIithLkNVUlNPUl9DTEFTUysiLiIrYS5DVVJTT1JfQkxJTktfQ0xBU1MrIi4iK2EuQ1VSU09SX1NUWUxFX0JMT0NLX0NMQVNTKyIgeyBhbmltYXRpb246IGJsaW5rX2Jsb2NrXyIrdGhpcy5fdGVybWluYWxDbGFzcysiIDFzIHN0ZXAtZW5kIGluZmluaXRlO30iK3RoaXMuX3Rlcm1pbmFsU2VsZWN0b3IrIiAueHRlcm0tcm93cy54dGVybS1mb2N1cyAuIithLkNVUlNPUl9DTEFTUysiLiIrYS5DVVJTT1JfU1RZTEVfQkxPQ0tfQ0xBU1MrIiB7IGJhY2tncm91bmQtY29sb3I6ICIrdGhpcy5fY29sb3JzLmN1cnNvci5jc3MrIjsgY29sb3I6ICIrdGhpcy5fY29sb3JzLmN1cnNvckFjY2VudC5jc3MrIjt9Iit0aGlzLl90ZXJtaW5hbFNlbGVjdG9yKyIgLnh0ZXJtLXJvd3MgLiIrYS5DVVJTT1JfQ0xBU1MrIi4iK2EuQ1VSU09SX1NUWUxFX0JBUl9DTEFTUysiIHsgYm94LXNoYWRvdzogIit0aGlzLl9vcHRpb25zU2VydmljZS5vcHRpb25zLmN1cnNvcldpZHRoKyJweCAwIDAgIit0aGlzLl9jb2xvcnMuY3Vyc29yLmNzcysiIGluc2V0O30iK3RoaXMuX3Rlcm1pbmFsU2VsZWN0b3IrIiAueHRlcm0tcm93cyAuIithLkNVUlNPUl9DTEFTUysiLiIrYS5DVVJTT1JfU1RZTEVfVU5ERVJMSU5FX0NMQVNTKyIgeyBib3gtc2hhZG93OiAwIC0xcHggMCAiK3RoaXMuX2NvbG9ycy5jdXJzb3IuY3NzKyIgaW5zZXQ7fSIsdCs9dGhpcy5fdGVybWluYWxTZWxlY3RvcisiIC54dGVybS1zZWxlY3Rpb24geyBwb3NpdGlvbjogYWJzb2x1dGU7IHRvcDogMDsgbGVmdDogMDsgei1pbmRleDogMTsgcG9pbnRlci1ldmVudHM6IG5vbmU7fSIrdGhpcy5fdGVybWluYWxTZWxlY3RvcisiIC54dGVybS1zZWxlY3Rpb24gZGl2IHsgcG9zaXRpb246IGFic29sdXRlOyBiYWNrZ3JvdW5kLWNvbG9yOiAiK3RoaXMuX2NvbG9ycy5zZWxlY3Rpb25UcmFuc3BhcmVudC5jc3MrIjt9Iix0aGlzLl9jb2xvcnMuYW5zaS5mb3JFYWNoKChmdW5jdGlvbihyLGkpe3QrPWUuX3Rlcm1pbmFsU2VsZWN0b3IrIiAuIit2K2krIiB7IGNvbG9yOiAiK3IuY3NzKyI7IH0iK2UuX3Rlcm1pbmFsU2VsZWN0b3IrIiAuIitnK2krIiB7IGJhY2tncm91bmQtY29sb3I6ICIrci5jc3MrIjsgfSJ9KSksdCs9dGhpcy5fdGVybWluYWxTZWxlY3RvcisiIC4iK3YrYy5JTlZFUlRFRF9ERUZBVUxUX0NPTE9SKyIgeyBjb2xvcjogIitfLmNvbG9yLm9wYXF1ZSh0aGlzLl9jb2xvcnMuYmFja2dyb3VuZCkuY3NzKyI7IH0iK3RoaXMuX3Rlcm1pbmFsU2VsZWN0b3IrIiAuIitnK2MuSU5WRVJURURfREVGQVVMVF9DT0xPUisiIHsgYmFja2dyb3VuZC1jb2xvcjogIit0aGlzLl9jb2xvcnMuZm9yZWdyb3VuZC5jc3MrIjsgfSIsdGhpcy5fdGhlbWVTdHlsZUVsZW1lbnQudGV4dENvbnRlbnQ9dH0sdC5wcm90b3R5cGUub25EZXZpY2VQaXhlbFJhdGlvQ2hhbmdlPWZ1bmN0aW9uKCl7dGhpcy5fdXBkYXRlRGltZW5zaW9ucygpfSx0LnByb3RvdHlwZS5fcmVmcmVzaFJvd0VsZW1lbnRzPWZ1bmN0aW9uKGUsdCl7Zm9yKHZhciByPXRoaXMuX3Jvd0VsZW1lbnRzLmxlbmd0aDtyPD10O3IrKyl7dmFyIGk9ZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgiZGl2Iik7dGhpcy5fcm93Q29udGFpbmVyLmFwcGVuZENoaWxkKGkpLHRoaXMuX3Jvd0VsZW1lbnRzLnB1c2goaSl9Zm9yKDt0aGlzLl9yb3dFbGVtZW50cy5sZW5ndGg+dDspdGhpcy5fcm93Q29udGFpbmVyLnJlbW92ZUNoaWxkKHRoaXMuX3Jvd0VsZW1lbnRzLnBvcCgpKX0sdC5wcm90b3R5cGUub25SZXNpemU9ZnVuY3Rpb24oZSx0KXt0aGlzLl9yZWZyZXNoUm93RWxlbWVudHMoZSx0KSx0aGlzLl91cGRhdGVEaW1lbnNpb25zKCl9LHQucHJvdG90eXBlLm9uQ2hhclNpemVDaGFuZ2VkPWZ1bmN0aW9uKCl7dGhpcy5fdXBkYXRlRGltZW5zaW9ucygpfSx0LnByb3RvdHlwZS5vbkJsdXI9ZnVuY3Rpb24oKXt0aGlzLl9yb3dDb250YWluZXIuY2xhc3NMaXN0LnJlbW92ZSh5KX0sdC5wcm90b3R5cGUub25Gb2N1cz1mdW5jdGlvbigpe3RoaXMuX3Jvd0NvbnRhaW5lci5jbGFzc0xpc3QuYWRkKHkpfSx0LnByb3RvdHlwZS5vblNlbGVjdGlvbkNoYW5nZWQ9ZnVuY3Rpb24oZSx0LHIpe2Zvcig7dGhpcy5fc2VsZWN0aW9uQ29udGFpbmVyLmNoaWxkcmVuLmxlbmd0aDspdGhpcy5fc2VsZWN0aW9uQ29udGFpbmVyLnJlbW92ZUNoaWxkKHRoaXMuX3NlbGVjdGlvbkNvbnRhaW5lci5jaGlsZHJlblswXSk7aWYoZSYmdCl7dmFyIGk9ZVsxXS10aGlzLl9idWZmZXJTZXJ2aWNlLmJ1ZmZlci55ZGlzcCxuPXRbMV0tdGhpcy5fYnVmZmVyU2VydmljZS5idWZmZXIueWRpc3Asbz1NYXRoLm1heChpLDApLHM9TWF0aC5taW4obix0aGlzLl9idWZmZXJTZXJ2aWNlLnJvd3MtMSk7aWYoIShvPj10aGlzLl9idWZmZXJTZXJ2aWNlLnJvd3N8fHM8MCkpe3ZhciBhPWRvY3VtZW50LmNyZWF0ZURvY3VtZW50RnJhZ21lbnQoKTtpZihyKWEuYXBwZW5kQ2hpbGQodGhpcy5fY3JlYXRlU2VsZWN0aW9uRWxlbWVudChvLGVbMF0sdFswXSxzLW8rMSkpO2Vsc2V7dmFyIGM9aT09PW8/ZVswXTowLGw9bz09PW4/dFswXTp0aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHM7YS5hcHBlbmRDaGlsZCh0aGlzLl9jcmVhdGVTZWxlY3Rpb25FbGVtZW50KG8sYyxsKSk7dmFyIHU9cy1vLTE7aWYoYS5hcHBlbmRDaGlsZCh0aGlzLl9jcmVhdGVTZWxlY3Rpb25FbGVtZW50KG8rMSwwLHRoaXMuX2J1ZmZlclNlcnZpY2UuY29scyx1KSksbyE9PXMpe3ZhciBoPW49PT1zP3RbMF06dGhpcy5fYnVmZmVyU2VydmljZS5jb2xzO2EuYXBwZW5kQ2hpbGQodGhpcy5fY3JlYXRlU2VsZWN0aW9uRWxlbWVudChzLDAsaCkpfX10aGlzLl9zZWxlY3Rpb25Db250YWluZXIuYXBwZW5kQ2hpbGQoYSl9fX0sdC5wcm90b3R5cGUuX2NyZWF0ZVNlbGVjdGlvbkVsZW1lbnQ9ZnVuY3Rpb24oZSx0LHIsaSl7dm9pZCAwPT09aSYmKGk9MSk7dmFyIG49ZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgiZGl2Iik7cmV0dXJuIG4uc3R5bGUuaGVpZ2h0PWkqdGhpcy5kaW1lbnNpb25zLmFjdHVhbENlbGxIZWlnaHQrInB4IixuLnN0eWxlLnRvcD1lKnRoaXMuZGltZW5zaW9ucy5hY3R1YWxDZWxsSGVpZ2h0KyJweCIsbi5zdHlsZS5sZWZ0PXQqdGhpcy5kaW1lbnNpb25zLmFjdHVhbENlbGxXaWR0aCsicHgiLG4uc3R5bGUud2lkdGg9dGhpcy5kaW1lbnNpb25zLmFjdHVhbENlbGxXaWR0aCooci10KSsicHgiLG59LHQucHJvdG90eXBlLm9uQ3Vyc29yTW92ZT1mdW5jdGlvbigpe30sdC5wcm90b3R5cGUub25PcHRpb25zQ2hhbmdlZD1mdW5jdGlvbigpe3RoaXMuX3VwZGF0ZURpbWVuc2lvbnMoKSx0aGlzLl9pbmplY3RDc3MoKX0sdC5wcm90b3R5cGUuY2xlYXI9ZnVuY3Rpb24oKXtmb3IodmFyIGU9MCx0PXRoaXMuX3Jvd0VsZW1lbnRzO2U8dC5sZW5ndGg7ZSsrKXRbZV0uaW5uZXJUZXh0PSIifSx0LnByb3RvdHlwZS5yZW5kZXJSb3dzPWZ1bmN0aW9uKGUsdCl7Zm9yKHZhciByPXRoaXMuX2J1ZmZlclNlcnZpY2UuYnVmZmVyLnliYXNlK3RoaXMuX2J1ZmZlclNlcnZpY2UuYnVmZmVyLnksaT1NYXRoLm1pbih0aGlzLl9idWZmZXJTZXJ2aWNlLmJ1ZmZlci54LHRoaXMuX2J1ZmZlclNlcnZpY2UuY29scy0xKSxuPXRoaXMuX29wdGlvbnNTZXJ2aWNlLm9wdGlvbnMuY3Vyc29yQmxpbmssbz1lO288PXQ7bysrKXt2YXIgcz10aGlzLl9yb3dFbGVtZW50c1tvXTtzLmlubmVyVGV4dD0iIjt2YXIgYT1vK3RoaXMuX2J1ZmZlclNlcnZpY2UuYnVmZmVyLnlkaXNwLGM9dGhpcy5fYnVmZmVyU2VydmljZS5idWZmZXIubGluZXMuZ2V0KGEpLGw9dGhpcy5fb3B0aW9uc1NlcnZpY2Uub3B0aW9ucy5jdXJzb3JTdHlsZTtzLmFwcGVuZENoaWxkKHRoaXMuX3Jvd0ZhY3RvcnkuY3JlYXRlUm93KGMsYSxhPT09cixsLGksbix0aGlzLmRpbWVuc2lvbnMuYWN0dWFsQ2VsbFdpZHRoLHRoaXMuX2J1ZmZlclNlcnZpY2UuY29scykpfX0sT2JqZWN0LmRlZmluZVByb3BlcnR5KHQucHJvdG90eXBlLCJfdGVybWluYWxTZWxlY3RvciIse2dldDpmdW5jdGlvbigpe3JldHVybiIuIitwK3RoaXMuX3Rlcm1pbmFsQ2xhc3N9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksdC5wcm90b3R5cGUuX29uTGlua0hvdmVyPWZ1bmN0aW9uKGUpe3RoaXMuX3NldENlbGxVbmRlcmxpbmUoZS54MSxlLngyLGUueTEsZS55MixlLmNvbHMsITApfSx0LnByb3RvdHlwZS5fb25MaW5rTGVhdmU9ZnVuY3Rpb24oZSl7dGhpcy5fc2V0Q2VsbFVuZGVybGluZShlLngxLGUueDIsZS55MSxlLnkyLGUuY29scywhMSl9LHQucHJvdG90eXBlLl9zZXRDZWxsVW5kZXJsaW5lPWZ1bmN0aW9uKGUsdCxyLGksbixvKXtmb3IoO2UhPT10fHxyIT09aTspe3ZhciBzPXRoaXMuX3Jvd0VsZW1lbnRzW3JdO2lmKCFzKXJldHVybjt2YXIgYT1zLmNoaWxkcmVuW2VdO2EmJihhLnN0eWxlLnRleHREZWNvcmF0aW9uPW8/InVuZGVybGluZSI6Im5vbmUiKSwrK2U+PW4mJihlPTAscisrKX19LG8oW3MoNixoLklJbnN0YW50aWF0aW9uU2VydmljZSkscyg3LHUuSUNoYXJTaXplU2VydmljZSkscyg4LGguSU9wdGlvbnNTZXJ2aWNlKSxzKDksaC5JQnVmZmVyU2VydmljZSldLHQpfShsLkRpc3Bvc2FibGUpO3QuRG9tUmVuZGVyZXI9Yn0sMzc4NzpmdW5jdGlvbihlLHQscil7dmFyIGk9dGhpcyYmdGhpcy5fX2RlY29yYXRlfHxmdW5jdGlvbihlLHQscixpKXt2YXIgbixvPWFyZ3VtZW50cy5sZW5ndGgscz1vPDM/dDpudWxsPT09aT9pPU9iamVjdC5nZXRPd25Qcm9wZXJ0eURlc2NyaXB0b3IodCxyKTppO2lmKCJvYmplY3QiPT10eXBlb2YgUmVmbGVjdCYmImZ1bmN0aW9uIj09dHlwZW9mIFJlZmxlY3QuZGVjb3JhdGUpcz1SZWZsZWN0LmRlY29yYXRlKGUsdCxyLGkpO2Vsc2UgZm9yKHZhciBhPWUubGVuZ3RoLTE7YT49MDthLS0pKG49ZVthXSkmJihzPShvPDM/bihzKTpvPjM/bih0LHIscyk6bih0LHIpKXx8cyk7cmV0dXJuIG8+MyYmcyYmT2JqZWN0LmRlZmluZVByb3BlcnR5KHQscixzKSxzfSxuPXRoaXMmJnRoaXMuX19wYXJhbXx8ZnVuY3Rpb24oZSx0KXtyZXR1cm4gZnVuY3Rpb24ocixpKXt0KHIsaSxlKX19O09iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LCJfX2VzTW9kdWxlIix7dmFsdWU6ITB9KSx0LkRvbVJlbmRlcmVyUm93RmFjdG9yeT10LkNVUlNPUl9TVFlMRV9VTkRFUkxJTkVfQ0xBU1M9dC5DVVJTT1JfU1RZTEVfQkFSX0NMQVNTPXQuQ1VSU09SX1NUWUxFX0JMT0NLX0NMQVNTPXQuQ1VSU09SX0JMSU5LX0NMQVNTPXQuQ1VSU09SX0NMQVNTPXQuU1RSSUtFVEhST1VHSF9DTEFTUz10LlVOREVSTElORV9DTEFTUz10LklUQUxJQ19DTEFTUz10LkRJTV9DTEFTUz10LkJPTERfQ0xBU1M9dm9pZCAwO3ZhciBvPXIoODgwMykscz1yKDY0MyksYT1yKDUxMSksYz1yKDI1ODUpLGw9cig0Nzc0KSx1PXIoNDcyNSksaD1yKDQyNjkpO3QuQk9MRF9DTEFTUz0ieHRlcm0tYm9sZCIsdC5ESU1fQ0xBU1M9Inh0ZXJtLWRpbSIsdC5JVEFMSUNfQ0xBU1M9Inh0ZXJtLWl0YWxpYyIsdC5VTkRFUkxJTkVfQ0xBU1M9Inh0ZXJtLXVuZGVybGluZSIsdC5TVFJJS0VUSFJPVUdIX0NMQVNTPSJ4dGVybS1zdHJpa2V0aHJvdWdoIix0LkNVUlNPUl9DTEFTUz0ieHRlcm0tY3Vyc29yIix0LkNVUlNPUl9CTElOS19DTEFTUz0ieHRlcm0tY3Vyc29yLWJsaW5rIix0LkNVUlNPUl9TVFlMRV9CTE9DS19DTEFTUz0ieHRlcm0tY3Vyc29yLWJsb2NrIix0LkNVUlNPUl9TVFlMRV9CQVJfQ0xBU1M9Inh0ZXJtLWN1cnNvci1iYXIiLHQuQ1VSU09SX1NUWUxFX1VOREVSTElORV9DTEFTUz0ieHRlcm0tY3Vyc29yLXVuZGVybGluZSI7dmFyIGY9ZnVuY3Rpb24oKXtmdW5jdGlvbiBlKGUsdCxyLGksbil7dGhpcy5fZG9jdW1lbnQ9ZSx0aGlzLl9jb2xvcnM9dCx0aGlzLl9jaGFyYWN0ZXJKb2luZXJTZXJ2aWNlPXIsdGhpcy5fb3B0aW9uc1NlcnZpY2U9aSx0aGlzLl9jb3JlU2VydmljZT1uLHRoaXMuX3dvcmtDZWxsPW5ldyBhLkNlbGxEYXRhfXJldHVybiBlLnByb3RvdHlwZS5zZXRDb2xvcnM9ZnVuY3Rpb24oZSl7dGhpcy5fY29sb3JzPWV9LGUucHJvdG90eXBlLmNyZWF0ZVJvdz1mdW5jdGlvbihlLHIsaSxuLGEsYyx1LGYpe2Zvcih2YXIgZD10aGlzLl9kb2N1bWVudC5jcmVhdGVEb2N1bWVudEZyYWdtZW50KCkscD10aGlzLl9jaGFyYWN0ZXJKb2luZXJTZXJ2aWNlLmdldEpvaW5lZENoYXJhY3RlcnMociksdj0wLGc9TWF0aC5taW4oZS5sZW5ndGgsZiktMTtnPj0wO2ctLSlpZihlLmxvYWRDZWxsKGcsdGhpcy5fd29ya0NlbGwpLmdldENvZGUoKSE9PXMuTlVMTF9DRUxMX0NPREV8fGkmJmc9PT1hKXt2PWcrMTticmVha31mb3IoZz0wO2c8djtnKyspe2UubG9hZENlbGwoZyx0aGlzLl93b3JrQ2VsbCk7dmFyIHk9dGhpcy5fd29ya0NlbGwuZ2V0V2lkdGgoKTtpZigwIT09eSl7dmFyIG09ITEsYj1nLFM9dGhpcy5fd29ya0NlbGw7aWYocC5sZW5ndGg+MCYmZz09PXBbMF1bMF0pe209ITA7dmFyIEM9cC5zaGlmdCgpO1M9bmV3IGguSm9pbmVkQ2VsbERhdGEodGhpcy5fd29ya0NlbGwsZS50cmFuc2xhdGVUb1N0cmluZyghMCxDWzBdLENbMV0pLENbMV0tQ1swXSksYj1DWzFdLTEseT1TLmdldFdpZHRoKCl9dmFyIHc9dGhpcy5fZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgic3BhbiIpO2lmKHk+MSYmKHcuc3R5bGUud2lkdGg9dSp5KyJweCIpLG0mJih3LnN0eWxlLmRpc3BsYXk9ImlubGluZSIsYT49ZyYmYTw9YiYmKGE9ZykpLCF0aGlzLl9jb3JlU2VydmljZS5pc0N1cnNvckhpZGRlbiYmaSYmZz09PWEpc3dpdGNoKHcuY2xhc3NMaXN0LmFkZCh0LkNVUlNPUl9DTEFTUyksYyYmdy5jbGFzc0xpc3QuYWRkKHQuQ1VSU09SX0JMSU5LX0NMQVNTKSxuKXtjYXNlImJhciI6dy5jbGFzc0xpc3QuYWRkKHQuQ1VSU09SX1NUWUxFX0JBUl9DTEFTUyk7YnJlYWs7Y2FzZSJ1bmRlcmxpbmUiOncuY2xhc3NMaXN0LmFkZCh0LkNVUlNPUl9TVFlMRV9VTkRFUkxJTkVfQ0xBU1MpO2JyZWFrO2RlZmF1bHQ6dy5jbGFzc0xpc3QuYWRkKHQuQ1VSU09SX1NUWUxFX0JMT0NLX0NMQVNTKX1TLmlzQm9sZCgpJiZ3LmNsYXNzTGlzdC5hZGQodC5CT0xEX0NMQVNTKSxTLmlzSXRhbGljKCkmJncuY2xhc3NMaXN0LmFkZCh0LklUQUxJQ19DTEFTUyksUy5pc0RpbSgpJiZ3LmNsYXNzTGlzdC5hZGQodC5ESU1fQ0xBU1MpLFMuaXNVbmRlcmxpbmUoKSYmdy5jbGFzc0xpc3QuYWRkKHQuVU5ERVJMSU5FX0NMQVNTKSxTLmlzSW52aXNpYmxlKCk/dy50ZXh0Q29udGVudD1zLldISVRFU1BBQ0VfQ0VMTF9DSEFSOncudGV4dENvbnRlbnQ9Uy5nZXRDaGFycygpfHxzLldISVRFU1BBQ0VfQ0VMTF9DSEFSLFMuaXNTdHJpa2V0aHJvdWdoKCkmJncuY2xhc3NMaXN0LmFkZCh0LlNUUklLRVRIUk9VR0hfQ0xBU1MpO3ZhciBMPVMuZ2V0RmdDb2xvcigpLEU9Uy5nZXRGZ0NvbG9yTW9kZSgpLHg9Uy5nZXRCZ0NvbG9yKCksQT1TLmdldEJnQ29sb3JNb2RlKCksaz0hIVMuaXNJbnZlcnNlKCk7aWYoayl7dmFyIE09TDtMPXgseD1NO3ZhciBSPUU7RT1BLEE9Un1zd2l0Y2goRSl7Y2FzZSAxNjc3NzIxNjpjYXNlIDMzNTU0NDMyOlMuaXNCb2xkKCkmJkw8OCYmdGhpcy5fb3B0aW9uc1NlcnZpY2Uub3B0aW9ucy5kcmF3Qm9sZFRleHRJbkJyaWdodENvbG9ycyYmKEwrPTgpLHRoaXMuX2FwcGx5TWluaW11bUNvbnRyYXN0KHcsdGhpcy5fY29sb3JzLmJhY2tncm91bmQsdGhpcy5fY29sb3JzLmFuc2lbTF0pfHx3LmNsYXNzTGlzdC5hZGQoInh0ZXJtLWZnLSIrTCk7YnJlYWs7Y2FzZSA1MDMzMTY0ODp2YXIgVD1sLnJnYmEudG9Db2xvcihMPj4xNiYyNTUsTD4+OCYyNTUsMjU1JkwpO3RoaXMuX2FwcGx5TWluaW11bUNvbnRyYXN0KHcsdGhpcy5fY29sb3JzLmJhY2tncm91bmQsVCl8fHRoaXMuX2FkZFN0eWxlKHcsImNvbG9yOiMiK18oTC50b1N0cmluZygxNiksIjAiLDYpKTticmVhaztkZWZhdWx0OnRoaXMuX2FwcGx5TWluaW11bUNvbnRyYXN0KHcsdGhpcy5fY29sb3JzLmJhY2tncm91bmQsdGhpcy5fY29sb3JzLmZvcmVncm91bmQpfHxrJiZ3LmNsYXNzTGlzdC5hZGQoInh0ZXJtLWZnLSIrby5JTlZFUlRFRF9ERUZBVUxUX0NPTE9SKX1zd2l0Y2goQSl7Y2FzZSAxNjc3NzIxNjpjYXNlIDMzNTU0NDMyOncuY2xhc3NMaXN0LmFkZCgieHRlcm0tYmctIit4KTticmVhaztjYXNlIDUwMzMxNjQ4OnRoaXMuX2FkZFN0eWxlKHcsImJhY2tncm91bmQtY29sb3I6IyIrXyh4LnRvU3RyaW5nKDE2KSwiMCIsNikpO2JyZWFrO2RlZmF1bHQ6ayYmdy5jbGFzc0xpc3QuYWRkKCJ4dGVybS1iZy0iK28uSU5WRVJURURfREVGQVVMVF9DT0xPUil9ZC5hcHBlbmRDaGlsZCh3KSxnPWJ9fXJldHVybiBkfSxlLnByb3RvdHlwZS5fYXBwbHlNaW5pbXVtQ29udHJhc3Q9ZnVuY3Rpb24oZSx0LHIpe2lmKDE9PT10aGlzLl9vcHRpb25zU2VydmljZS5vcHRpb25zLm1pbmltdW1Db250cmFzdFJhdGlvKXJldHVybiExO3ZhciBpPXRoaXMuX2NvbG9ycy5jb250cmFzdENhY2hlLmdldENvbG9yKHRoaXMuX3dvcmtDZWxsLmJnLHRoaXMuX3dvcmtDZWxsLmZnKTtyZXR1cm4gdm9pZCAwPT09aSYmKGk9bC5jb2xvci5lbnN1cmVDb250cmFzdFJhdGlvKHQscix0aGlzLl9vcHRpb25zU2VydmljZS5vcHRpb25zLm1pbmltdW1Db250cmFzdFJhdGlvKSx0aGlzLl9jb2xvcnMuY29udHJhc3RDYWNoZS5zZXRDb2xvcih0aGlzLl93b3JrQ2VsbC5iZyx0aGlzLl93b3JrQ2VsbC5mZyxudWxsIT1pP2k6bnVsbCkpLCEhaSYmKHRoaXMuX2FkZFN0eWxlKGUsImNvbG9yOiIraS5jc3MpLCEwKX0sZS5wcm90b3R5cGUuX2FkZFN0eWxlPWZ1bmN0aW9uKGUsdCl7ZS5zZXRBdHRyaWJ1dGUoInN0eWxlIiwiIisoZS5nZXRBdHRyaWJ1dGUoInN0eWxlIil8fCIiKSt0KyI7Iil9LGkoW24oMix1LklDaGFyYWN0ZXJKb2luZXJTZXJ2aWNlKSxuKDMsYy5JT3B0aW9uc1NlcnZpY2UpLG4oNCxjLklDb3JlU2VydmljZSldLGUpfSgpO2Z1bmN0aW9uIF8oZSx0LHIpe2Zvcig7ZS5sZW5ndGg8cjspZT10K2U7cmV0dXJuIGV9dC5Eb21SZW5kZXJlclJvd0ZhY3Rvcnk9Zn0sNDU2OihlLHQpPT57T2JqZWN0LmRlZmluZVByb3BlcnR5KHQsIl9fZXNNb2R1bGUiLHt2YWx1ZTohMH0pLHQuU2VsZWN0aW9uTW9kZWw9dm9pZCAwO3ZhciByPWZ1bmN0aW9uKCl7ZnVuY3Rpb24gZShlKXt0aGlzLl9idWZmZXJTZXJ2aWNlPWUsdGhpcy5pc1NlbGVjdEFsbEFjdGl2ZT0hMSx0aGlzLnNlbGVjdGlvblN0YXJ0TGVuZ3RoPTB9cmV0dXJuIGUucHJvdG90eXBlLmNsZWFyU2VsZWN0aW9uPWZ1bmN0aW9uKCl7dGhpcy5zZWxlY3Rpb25TdGFydD12b2lkIDAsdGhpcy5zZWxlY3Rpb25FbmQ9dm9pZCAwLHRoaXMuaXNTZWxlY3RBbGxBY3RpdmU9ITEsdGhpcy5zZWxlY3Rpb25TdGFydExlbmd0aD0wfSxPYmplY3QuZGVmaW5lUHJvcGVydHkoZS5wcm90b3R5cGUsImZpbmFsU2VsZWN0aW9uU3RhcnQiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5pc1NlbGVjdEFsbEFjdGl2ZT9bMCwwXTp0aGlzLnNlbGVjdGlvbkVuZCYmdGhpcy5zZWxlY3Rpb25TdGFydCYmdGhpcy5hcmVTZWxlY3Rpb25WYWx1ZXNSZXZlcnNlZCgpP3RoaXMuc2VsZWN0aW9uRW5kOnRoaXMuc2VsZWN0aW9uU3RhcnR9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KGUucHJvdG90eXBlLCJmaW5hbFNlbGVjdGlvbkVuZCIse2dldDpmdW5jdGlvbigpe2lmKHRoaXMuaXNTZWxlY3RBbGxBY3RpdmUpcmV0dXJuW3RoaXMuX2J1ZmZlclNlcnZpY2UuY29scyx0aGlzLl9idWZmZXJTZXJ2aWNlLmJ1ZmZlci55YmFzZSt0aGlzLl9idWZmZXJTZXJ2aWNlLnJvd3MtMV07aWYodGhpcy5zZWxlY3Rpb25TdGFydCl7aWYoIXRoaXMuc2VsZWN0aW9uRW5kfHx0aGlzLmFyZVNlbGVjdGlvblZhbHVlc1JldmVyc2VkKCkpe3ZhciBlPXRoaXMuc2VsZWN0aW9uU3RhcnRbMF0rdGhpcy5zZWxlY3Rpb25TdGFydExlbmd0aDtyZXR1cm4gZT50aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHM/ZSV0aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHM9PTA/W3RoaXMuX2J1ZmZlclNlcnZpY2UuY29scyx0aGlzLnNlbGVjdGlvblN0YXJ0WzFdK01hdGguZmxvb3IoZS90aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHMpLTFdOltlJXRoaXMuX2J1ZmZlclNlcnZpY2UuY29scyx0aGlzLnNlbGVjdGlvblN0YXJ0WzFdK01hdGguZmxvb3IoZS90aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHMpXTpbZSx0aGlzLnNlbGVjdGlvblN0YXJ0WzFdXX1yZXR1cm4gdGhpcy5zZWxlY3Rpb25TdGFydExlbmd0aCYmdGhpcy5zZWxlY3Rpb25FbmRbMV09PT10aGlzLnNlbGVjdGlvblN0YXJ0WzFdP1tNYXRoLm1heCh0aGlzLnNlbGVjdGlvblN0YXJ0WzBdK3RoaXMuc2VsZWN0aW9uU3RhcnRMZW5ndGgsdGhpcy5zZWxlY3Rpb25FbmRbMF0pLHRoaXMuc2VsZWN0aW9uRW5kWzFdXTp0aGlzLnNlbGVjdGlvbkVuZH19LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksZS5wcm90b3R5cGUuYXJlU2VsZWN0aW9uVmFsdWVzUmV2ZXJzZWQ9ZnVuY3Rpb24oKXt2YXIgZT10aGlzLnNlbGVjdGlvblN0YXJ0LHQ9dGhpcy5zZWxlY3Rpb25FbmQ7cmV0dXJuISghZXx8IXQpJiYoZVsxXT50WzFdfHxlWzFdPT09dFsxXSYmZVswXT50WzBdKX0sZS5wcm90b3R5cGUub25UcmltPWZ1bmN0aW9uKGUpe3JldHVybiB0aGlzLnNlbGVjdGlvblN0YXJ0JiYodGhpcy5zZWxlY3Rpb25TdGFydFsxXS09ZSksdGhpcy5zZWxlY3Rpb25FbmQmJih0aGlzLnNlbGVjdGlvbkVuZFsxXS09ZSksdGhpcy5zZWxlY3Rpb25FbmQmJnRoaXMuc2VsZWN0aW9uRW5kWzFdPDA/KHRoaXMuY2xlYXJTZWxlY3Rpb24oKSwhMCk6KHRoaXMuc2VsZWN0aW9uU3RhcnQmJnRoaXMuc2VsZWN0aW9uU3RhcnRbMV08MCYmKHRoaXMuc2VsZWN0aW9uU3RhcnRbMV09MCksITEpfSxlfSgpO3QuU2VsZWN0aW9uTW9kZWw9cn0sNDI4OmZ1bmN0aW9uKGUsdCxyKXt2YXIgaT10aGlzJiZ0aGlzLl9fZGVjb3JhdGV8fGZ1bmN0aW9uKGUsdCxyLGkpe3ZhciBuLG89YXJndW1lbnRzLmxlbmd0aCxzPW88Mz90Om51bGw9PT1pP2k9T2JqZWN0LmdldE93blByb3BlcnR5RGVzY3JpcHRvcih0LHIpOmk7aWYoIm9iamVjdCI9PXR5cGVvZiBSZWZsZWN0JiYiZnVuY3Rpb24iPT10eXBlb2YgUmVmbGVjdC5kZWNvcmF0ZSlzPVJlZmxlY3QuZGVjb3JhdGUoZSx0LHIsaSk7ZWxzZSBmb3IodmFyIGE9ZS5sZW5ndGgtMTthPj0wO2EtLSkobj1lW2FdKSYmKHM9KG88Mz9uKHMpOm8+Mz9uKHQscixzKTpuKHQscikpfHxzKTtyZXR1cm4gbz4zJiZzJiZPYmplY3QuZGVmaW5lUHJvcGVydHkodCxyLHMpLHN9LG49dGhpcyYmdGhpcy5fX3BhcmFtfHxmdW5jdGlvbihlLHQpe3JldHVybiBmdW5jdGlvbihyLGkpe3QocixpLGUpfX07T2JqZWN0LmRlZmluZVByb3BlcnR5KHQsIl9fZXNNb2R1bGUiLHt2YWx1ZTohMH0pLHQuQ2hhclNpemVTZXJ2aWNlPXZvaWQgMDt2YXIgbz1yKDI1ODUpLHM9cig4NDYwKSxhPWZ1bmN0aW9uKCl7ZnVuY3Rpb24gZShlLHQscil7dGhpcy5fb3B0aW9uc1NlcnZpY2U9cix0aGlzLndpZHRoPTAsdGhpcy5oZWlnaHQ9MCx0aGlzLl9vbkNoYXJTaXplQ2hhbmdlPW5ldyBzLkV2ZW50RW1pdHRlcix0aGlzLl9tZWFzdXJlU3RyYXRlZ3k9bmV3IGMoZSx0LHRoaXMuX29wdGlvbnNTZXJ2aWNlKX1yZXR1cm4gT2JqZWN0LmRlZmluZVByb3BlcnR5KGUucHJvdG90eXBlLCJoYXNWYWxpZFNpemUiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy53aWR0aD4wJiZ0aGlzLmhlaWdodD4wfSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLE9iamVjdC5kZWZpbmVQcm9wZXJ0eShlLnByb3RvdHlwZSwib25DaGFyU2l6ZUNoYW5nZSIse2dldDpmdW5jdGlvbigpe3JldHVybiB0aGlzLl9vbkNoYXJTaXplQ2hhbmdlLmV2ZW50fSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLGUucHJvdG90eXBlLm1lYXN1cmU9ZnVuY3Rpb24oKXt2YXIgZT10aGlzLl9tZWFzdXJlU3RyYXRlZ3kubWVhc3VyZSgpO2Uud2lkdGg9PT10aGlzLndpZHRoJiZlLmhlaWdodD09PXRoaXMuaGVpZ2h0fHwodGhpcy53aWR0aD1lLndpZHRoLHRoaXMuaGVpZ2h0PWUuaGVpZ2h0LHRoaXMuX29uQ2hhclNpemVDaGFuZ2UuZmlyZSgpKX0saShbbigyLG8uSU9wdGlvbnNTZXJ2aWNlKV0sZSl9KCk7dC5DaGFyU2l6ZVNlcnZpY2U9YTt2YXIgYz1mdW5jdGlvbigpe2Z1bmN0aW9uIGUoZSx0LHIpe3RoaXMuX2RvY3VtZW50PWUsdGhpcy5fcGFyZW50RWxlbWVudD10LHRoaXMuX29wdGlvbnNTZXJ2aWNlPXIsdGhpcy5fcmVzdWx0PXt3aWR0aDowLGhlaWdodDowfSx0aGlzLl9tZWFzdXJlRWxlbWVudD10aGlzLl9kb2N1bWVudC5jcmVhdGVFbGVtZW50KCJzcGFuIiksdGhpcy5fbWVhc3VyZUVsZW1lbnQuY2xhc3NMaXN0LmFkZCgieHRlcm0tY2hhci1tZWFzdXJlLWVsZW1lbnQiKSx0aGlzLl9tZWFzdXJlRWxlbWVudC50ZXh0Q29udGVudD0iVyIsdGhpcy5fbWVhc3VyZUVsZW1lbnQuc2V0QXR0cmlidXRlKCJhcmlhLWhpZGRlbiIsInRydWUiKSx0aGlzLl9wYXJlbnRFbGVtZW50LmFwcGVuZENoaWxkKHRoaXMuX21lYXN1cmVFbGVtZW50KX1yZXR1cm4gZS5wcm90b3R5cGUubWVhc3VyZT1mdW5jdGlvbigpe3RoaXMuX21lYXN1cmVFbGVtZW50LnN0eWxlLmZvbnRGYW1pbHk9dGhpcy5fb3B0aW9uc1NlcnZpY2Uub3B0aW9ucy5mb250RmFtaWx5LHRoaXMuX21lYXN1cmVFbGVtZW50LnN0eWxlLmZvbnRTaXplPXRoaXMuX29wdGlvbnNTZXJ2aWNlLm9wdGlvbnMuZm9udFNpemUrInB4Ijt2YXIgZT10aGlzLl9tZWFzdXJlRWxlbWVudC5nZXRCb3VuZGluZ0NsaWVudFJlY3QoKTtyZXR1cm4gMCE9PWUud2lkdGgmJjAhPT1lLmhlaWdodCYmKHRoaXMuX3Jlc3VsdC53aWR0aD1lLndpZHRoLHRoaXMuX3Jlc3VsdC5oZWlnaHQ9TWF0aC5jZWlsKGUuaGVpZ2h0KSksdGhpcy5fcmVzdWx0fSxlfSgpfSw0MjY5OmZ1bmN0aW9uKGUsdCxyKXt2YXIgaSxuPXRoaXMmJnRoaXMuX19leHRlbmRzfHwoaT1mdW5jdGlvbihlLHQpe3JldHVybiBpPU9iamVjdC5zZXRQcm90b3R5cGVPZnx8e19fcHJvdG9fXzpbXX1pbnN0YW5jZW9mIEFycmF5JiZmdW5jdGlvbihlLHQpe2UuX19wcm90b19fPXR9fHxmdW5jdGlvbihlLHQpe2Zvcih2YXIgciBpbiB0KU9iamVjdC5wcm90b3R5cGUuaGFzT3duUHJvcGVydHkuY2FsbCh0LHIpJiYoZVtyXT10W3JdKX0saShlLHQpfSxmdW5jdGlvbihlLHQpe2lmKCJmdW5jdGlvbiIhPXR5cGVvZiB0JiZudWxsIT09dCl0aHJvdyBuZXcgVHlwZUVycm9yKCJDbGFzcyBleHRlbmRzIHZhbHVlICIrU3RyaW5nKHQpKyIgaXMgbm90IGEgY29uc3RydWN0b3Igb3IgbnVsbCIpO2Z1bmN0aW9uIHIoKXt0aGlzLmNvbnN0cnVjdG9yPWV9aShlLHQpLGUucHJvdG90eXBlPW51bGw9PT10P09iamVjdC5jcmVhdGUodCk6KHIucHJvdG90eXBlPXQucHJvdG90eXBlLG5ldyByKX0pLG89dGhpcyYmdGhpcy5fX2RlY29yYXRlfHxmdW5jdGlvbihlLHQscixpKXt2YXIgbixvPWFyZ3VtZW50cy5sZW5ndGgscz1vPDM/dDpudWxsPT09aT9pPU9iamVjdC5nZXRPd25Qcm9wZXJ0eURlc2NyaXB0b3IodCxyKTppO2lmKCJvYmplY3QiPT10eXBlb2YgUmVmbGVjdCYmImZ1bmN0aW9uIj09dHlwZW9mIFJlZmxlY3QuZGVjb3JhdGUpcz1SZWZsZWN0LmRlY29yYXRlKGUsdCxyLGkpO2Vsc2UgZm9yKHZhciBhPWUubGVuZ3RoLTE7YT49MDthLS0pKG49ZVthXSkmJihzPShvPDM/bihzKTpvPjM/bih0LHIscyk6bih0LHIpKXx8cyk7cmV0dXJuIG8+MyYmcyYmT2JqZWN0LmRlZmluZVByb3BlcnR5KHQscixzKSxzfSxzPXRoaXMmJnRoaXMuX19wYXJhbXx8ZnVuY3Rpb24oZSx0KXtyZXR1cm4gZnVuY3Rpb24ocixpKXt0KHIsaSxlKX19O09iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LCJfX2VzTW9kdWxlIix7dmFsdWU6ITB9KSx0LkNoYXJhY3RlckpvaW5lclNlcnZpY2U9dC5Kb2luZWRDZWxsRGF0YT12b2lkIDA7dmFyIGE9cigzNzM0KSxjPXIoNjQzKSxsPXIoNTExKSx1PXIoMjU4NSksaD1mdW5jdGlvbihlKXtmdW5jdGlvbiB0KHQscixpKXt2YXIgbj1lLmNhbGwodGhpcyl8fHRoaXM7cmV0dXJuIG4uY29udGVudD0wLG4uY29tYmluZWREYXRhPSIiLG4uZmc9dC5mZyxuLmJnPXQuYmcsbi5jb21iaW5lZERhdGE9cixuLl93aWR0aD1pLG59cmV0dXJuIG4odCxlKSx0LnByb3RvdHlwZS5pc0NvbWJpbmVkPWZ1bmN0aW9uKCl7cmV0dXJuIDIwOTcxNTJ9LHQucHJvdG90eXBlLmdldFdpZHRoPWZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX3dpZHRofSx0LnByb3RvdHlwZS5nZXRDaGFycz1mdW5jdGlvbigpe3JldHVybiB0aGlzLmNvbWJpbmVkRGF0YX0sdC5wcm90b3R5cGUuZ2V0Q29kZT1mdW5jdGlvbigpe3JldHVybiAyMDk3MTUxfSx0LnByb3RvdHlwZS5zZXRGcm9tQ2hhckRhdGE9ZnVuY3Rpb24oZSl7dGhyb3cgbmV3IEVycm9yKCJub3QgaW1wbGVtZW50ZWQiKX0sdC5wcm90b3R5cGUuZ2V0QXNDaGFyRGF0YT1mdW5jdGlvbigpe3JldHVyblt0aGlzLmZnLHRoaXMuZ2V0Q2hhcnMoKSx0aGlzLmdldFdpZHRoKCksdGhpcy5nZXRDb2RlKCldfSx0fShhLkF0dHJpYnV0ZURhdGEpO3QuSm9pbmVkQ2VsbERhdGE9aDt2YXIgZj1mdW5jdGlvbigpe2Z1bmN0aW9uIGUoZSl7dGhpcy5fYnVmZmVyU2VydmljZT1lLHRoaXMuX2NoYXJhY3RlckpvaW5lcnM9W10sdGhpcy5fbmV4dENoYXJhY3RlckpvaW5lcklkPTAsdGhpcy5fd29ya0NlbGw9bmV3IGwuQ2VsbERhdGF9cmV0dXJuIGUucHJvdG90eXBlLnJlZ2lzdGVyPWZ1bmN0aW9uKGUpe3ZhciB0PXtpZDp0aGlzLl9uZXh0Q2hhcmFjdGVySm9pbmVySWQrKyxoYW5kbGVyOmV9O3JldHVybiB0aGlzLl9jaGFyYWN0ZXJKb2luZXJzLnB1c2godCksdC5pZH0sZS5wcm90b3R5cGUuZGVyZWdpc3Rlcj1mdW5jdGlvbihlKXtmb3IodmFyIHQ9MDt0PHRoaXMuX2NoYXJhY3RlckpvaW5lcnMubGVuZ3RoO3QrKylpZih0aGlzLl9jaGFyYWN0ZXJKb2luZXJzW3RdLmlkPT09ZSlyZXR1cm4gdGhpcy5fY2hhcmFjdGVySm9pbmVycy5zcGxpY2UodCwxKSwhMDtyZXR1cm4hMX0sZS5wcm90b3R5cGUuZ2V0Sm9pbmVkQ2hhcmFjdGVycz1mdW5jdGlvbihlKXtpZigwPT09dGhpcy5fY2hhcmFjdGVySm9pbmVycy5sZW5ndGgpcmV0dXJuW107dmFyIHQ9dGhpcy5fYnVmZmVyU2VydmljZS5idWZmZXIubGluZXMuZ2V0KGUpO2lmKCF0fHwwPT09dC5sZW5ndGgpcmV0dXJuW107Zm9yKHZhciByPVtdLGk9dC50cmFuc2xhdGVUb1N0cmluZyghMCksbj0wLG89MCxzPTAsYT10LmdldEZnKDApLGw9dC5nZXRCZygwKSx1PTA7dTx0LmdldFRyaW1tZWRMZW5ndGgoKTt1KyspaWYodC5sb2FkQ2VsbCh1LHRoaXMuX3dvcmtDZWxsKSwwIT09dGhpcy5fd29ya0NlbGwuZ2V0V2lkdGgoKSl7aWYodGhpcy5fd29ya0NlbGwuZmchPT1hfHx0aGlzLl93b3JrQ2VsbC5iZyE9PWwpe2lmKHUtbj4xKWZvcih2YXIgaD10aGlzLl9nZXRKb2luZWRSYW5nZXMoaSxzLG8sdCxuKSxmPTA7ZjxoLmxlbmd0aDtmKyspci5wdXNoKGhbZl0pO249dSxzPW8sYT10aGlzLl93b3JrQ2VsbC5mZyxsPXRoaXMuX3dvcmtDZWxsLmJnfW8rPXRoaXMuX3dvcmtDZWxsLmdldENoYXJzKCkubGVuZ3RofHxjLldISVRFU1BBQ0VfQ0VMTF9DSEFSLmxlbmd0aH1pZih0aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHMtbj4xKWZvcihoPXRoaXMuX2dldEpvaW5lZFJhbmdlcyhpLHMsbyx0LG4pLGY9MDtmPGgubGVuZ3RoO2YrKylyLnB1c2goaFtmXSk7cmV0dXJuIHJ9LGUucHJvdG90eXBlLl9nZXRKb2luZWRSYW5nZXM9ZnVuY3Rpb24odCxyLGksbixvKXt2YXIgcz10LnN1YnN0cmluZyhyLGkpLGE9W107dHJ5e2E9dGhpcy5fY2hhcmFjdGVySm9pbmVyc1swXS5oYW5kbGVyKHMpfWNhdGNoKGUpe2NvbnNvbGUuZXJyb3IoZSl9Zm9yKHZhciBjPTE7Yzx0aGlzLl9jaGFyYWN0ZXJKb2luZXJzLmxlbmd0aDtjKyspdHJ5e2Zvcih2YXIgbD10aGlzLl9jaGFyYWN0ZXJKb2luZXJzW2NdLmhhbmRsZXIocyksdT0wO3U8bC5sZW5ndGg7dSsrKWUuX21lcmdlUmFuZ2VzKGEsbFt1XSl9Y2F0Y2goZSl7Y29uc29sZS5lcnJvcihlKX1yZXR1cm4gdGhpcy5fc3RyaW5nUmFuZ2VzVG9DZWxsUmFuZ2VzKGEsbixvKSxhfSxlLnByb3RvdHlwZS5fc3RyaW5nUmFuZ2VzVG9DZWxsUmFuZ2VzPWZ1bmN0aW9uKGUsdCxyKXt2YXIgaT0wLG49ITEsbz0wLHM9ZVtpXTtpZihzKXtmb3IodmFyIGE9cjthPHRoaXMuX2J1ZmZlclNlcnZpY2UuY29sczthKyspe3ZhciBsPXQuZ2V0V2lkdGgoYSksdT10LmdldFN0cmluZyhhKS5sZW5ndGh8fGMuV0hJVEVTUEFDRV9DRUxMX0NIQVIubGVuZ3RoO2lmKDAhPT1sKXtpZighbiYmc1swXTw9byYmKHNbMF09YSxuPSEwKSxzWzFdPD1vKXtpZihzWzFdPWEsIShzPWVbKytpXSkpYnJlYWs7c1swXTw9bz8oc1swXT1hLG49ITApOm49ITF9bys9dX19cyYmKHNbMV09dGhpcy5fYnVmZmVyU2VydmljZS5jb2xzKX19LGUuX21lcmdlUmFuZ2VzPWZ1bmN0aW9uKGUsdCl7Zm9yKHZhciByPSExLGk9MDtpPGUubGVuZ3RoO2krKyl7dmFyIG49ZVtpXTtpZihyKXtpZih0WzFdPD1uWzBdKXJldHVybiBlW2ktMV1bMV09dFsxXSxlO2lmKHRbMV08PW5bMV0pcmV0dXJuIGVbaS0xXVsxXT1NYXRoLm1heCh0WzFdLG5bMV0pLGUuc3BsaWNlKGksMSksZTtlLnNwbGljZShpLDEpLGktLX1lbHNle2lmKHRbMV08PW5bMF0pcmV0dXJuIGUuc3BsaWNlKGksMCx0KSxlO2lmKHRbMV08PW5bMV0pcmV0dXJuIG5bMF09TWF0aC5taW4odFswXSxuWzBdKSxlO3RbMF08blsxXSYmKG5bMF09TWF0aC5taW4odFswXSxuWzBdKSxyPSEwKX19cmV0dXJuIHI/ZVtlLmxlbmd0aC0xXVsxXT10WzFdOmUucHVzaCh0KSxlfSxlPW8oW3MoMCx1LklCdWZmZXJTZXJ2aWNlKV0sZSl9KCk7dC5DaGFyYWN0ZXJKb2luZXJTZXJ2aWNlPWZ9LDUxMTQ6KGUsdCk9PntPYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC5Db3JlQnJvd3NlclNlcnZpY2U9dm9pZCAwO3ZhciByPWZ1bmN0aW9uKCl7ZnVuY3Rpb24gZShlKXt0aGlzLl90ZXh0YXJlYT1lfXJldHVybiBPYmplY3QuZGVmaW5lUHJvcGVydHkoZS5wcm90b3R5cGUsImlzRm9jdXNlZCIse2dldDpmdW5jdGlvbigpe3JldHVybih0aGlzLl90ZXh0YXJlYS5nZXRSb290Tm9kZT90aGlzLl90ZXh0YXJlYS5nZXRSb290Tm9kZSgpOmRvY3VtZW50KS5hY3RpdmVFbGVtZW50PT09dGhpcy5fdGV4dGFyZWEmJmRvY3VtZW50Lmhhc0ZvY3VzKCl9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksZX0oKTt0LkNvcmVCcm93c2VyU2VydmljZT1yfSw4OTM0OmZ1bmN0aW9uKGUsdCxyKXt2YXIgaT10aGlzJiZ0aGlzLl9fZGVjb3JhdGV8fGZ1bmN0aW9uKGUsdCxyLGkpe3ZhciBuLG89YXJndW1lbnRzLmxlbmd0aCxzPW88Mz90Om51bGw9PT1pP2k9T2JqZWN0LmdldE93blByb3BlcnR5RGVzY3JpcHRvcih0LHIpOmk7aWYoIm9iamVjdCI9PXR5cGVvZiBSZWZsZWN0JiYiZnVuY3Rpb24iPT10eXBlb2YgUmVmbGVjdC5kZWNvcmF0ZSlzPVJlZmxlY3QuZGVjb3JhdGUoZSx0LHIsaSk7ZWxzZSBmb3IodmFyIGE9ZS5sZW5ndGgtMTthPj0wO2EtLSkobj1lW2FdKSYmKHM9KG88Mz9uKHMpOm8+Mz9uKHQscixzKTpuKHQscikpfHxzKTtyZXR1cm4gbz4zJiZzJiZPYmplY3QuZGVmaW5lUHJvcGVydHkodCxyLHMpLHN9LG49dGhpcyYmdGhpcy5fX3BhcmFtfHxmdW5jdGlvbihlLHQpe3JldHVybiBmdW5jdGlvbihyLGkpe3QocixpLGUpfX07T2JqZWN0LmRlZmluZVByb3BlcnR5KHQsIl9fZXNNb2R1bGUiLHt2YWx1ZTohMH0pLHQuTW91c2VTZXJ2aWNlPXZvaWQgMDt2YXIgbz1yKDQ3MjUpLHM9cig5ODA2KSxhPWZ1bmN0aW9uKCl7ZnVuY3Rpb24gZShlLHQpe3RoaXMuX3JlbmRlclNlcnZpY2U9ZSx0aGlzLl9jaGFyU2l6ZVNlcnZpY2U9dH1yZXR1cm4gZS5wcm90b3R5cGUuZ2V0Q29vcmRzPWZ1bmN0aW9uKGUsdCxyLGksbil7cmV0dXJuKDAscy5nZXRDb29yZHMpKGUsdCxyLGksdGhpcy5fY2hhclNpemVTZXJ2aWNlLmhhc1ZhbGlkU2l6ZSx0aGlzLl9yZW5kZXJTZXJ2aWNlLmRpbWVuc2lvbnMuYWN0dWFsQ2VsbFdpZHRoLHRoaXMuX3JlbmRlclNlcnZpY2UuZGltZW5zaW9ucy5hY3R1YWxDZWxsSGVpZ2h0LG4pfSxlLnByb3RvdHlwZS5nZXRSYXdCeXRlQ29vcmRzPWZ1bmN0aW9uKGUsdCxyLGkpe3ZhciBuPXRoaXMuZ2V0Q29vcmRzKGUsdCxyLGkpO3JldHVybigwLHMuZ2V0UmF3Qnl0ZUNvb3Jkcykobil9LGkoW24oMCxvLklSZW5kZXJTZXJ2aWNlKSxuKDEsby5JQ2hhclNpemVTZXJ2aWNlKV0sZSl9KCk7dC5Nb3VzZVNlcnZpY2U9YX0sMzIzMDpmdW5jdGlvbihlLHQscil7dmFyIGksbj10aGlzJiZ0aGlzLl9fZXh0ZW5kc3x8KGk9ZnVuY3Rpb24oZSx0KXtyZXR1cm4gaT1PYmplY3Quc2V0UHJvdG90eXBlT2Z8fHtfX3Byb3RvX186W119aW5zdGFuY2VvZiBBcnJheSYmZnVuY3Rpb24oZSx0KXtlLl9fcHJvdG9fXz10fXx8ZnVuY3Rpb24oZSx0KXtmb3IodmFyIHIgaW4gdClPYmplY3QucHJvdG90eXBlLmhhc093blByb3BlcnR5LmNhbGwodCxyKSYmKGVbcl09dFtyXSl9LGkoZSx0KX0sZnVuY3Rpb24oZSx0KXtpZigiZnVuY3Rpb24iIT10eXBlb2YgdCYmbnVsbCE9PXQpdGhyb3cgbmV3IFR5cGVFcnJvcigiQ2xhc3MgZXh0ZW5kcyB2YWx1ZSAiK1N0cmluZyh0KSsiIGlzIG5vdCBhIGNvbnN0cnVjdG9yIG9yIG51bGwiKTtmdW5jdGlvbiByKCl7dGhpcy5jb25zdHJ1Y3Rvcj1lfWkoZSx0KSxlLnByb3RvdHlwZT1udWxsPT09dD9PYmplY3QuY3JlYXRlKHQpOihyLnByb3RvdHlwZT10LnByb3RvdHlwZSxuZXcgcil9KSxvPXRoaXMmJnRoaXMuX19kZWNvcmF0ZXx8ZnVuY3Rpb24oZSx0LHIsaSl7dmFyIG4sbz1hcmd1bWVudHMubGVuZ3RoLHM9bzwzP3Q6bnVsbD09PWk/aT1PYmplY3QuZ2V0T3duUHJvcGVydHlEZXNjcmlwdG9yKHQscik6aTtpZigib2JqZWN0Ij09dHlwZW9mIFJlZmxlY3QmJiJmdW5jdGlvbiI9PXR5cGVvZiBSZWZsZWN0LmRlY29yYXRlKXM9UmVmbGVjdC5kZWNvcmF0ZShlLHQscixpKTtlbHNlIGZvcih2YXIgYT1lLmxlbmd0aC0xO2E+PTA7YS0tKShuPWVbYV0pJiYocz0obzwzP24ocyk6bz4zP24odCxyLHMpOm4odCxyKSl8fHMpO3JldHVybiBvPjMmJnMmJk9iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LHIscyksc30scz10aGlzJiZ0aGlzLl9fcGFyYW18fGZ1bmN0aW9uKGUsdCl7cmV0dXJuIGZ1bmN0aW9uKHIsaSl7dChyLGksZSl9fTtPYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC5SZW5kZXJTZXJ2aWNlPXZvaWQgMDt2YXIgYT1yKDYxOTMpLGM9cig4NDYwKSxsPXIoODQ0KSx1PXIoNTU5NiksaD1yKDM2NTYpLGY9cigyNTg1KSxfPXIoNDcyNSksZD1mdW5jdGlvbihlKXtmdW5jdGlvbiB0KHQscixpLG4sbyxzKXt2YXIgbD1lLmNhbGwodGhpcyl8fHRoaXM7aWYobC5fcmVuZGVyZXI9dCxsLl9yb3dDb3VudD1yLGwuX2NoYXJTaXplU2VydmljZT1vLGwuX2lzUGF1c2VkPSExLGwuX25lZWRzRnVsbFJlZnJlc2g9ITEsbC5faXNOZXh0UmVuZGVyUmVkcmF3T25seT0hMCxsLl9uZWVkc1NlbGVjdGlvblJlZnJlc2g9ITEsbC5fY2FudmFzV2lkdGg9MCxsLl9jYW52YXNIZWlnaHQ9MCxsLl9zZWxlY3Rpb25TdGF0ZT17c3RhcnQ6dm9pZCAwLGVuZDp2b2lkIDAsY29sdW1uU2VsZWN0TW9kZTohMX0sbC5fb25EaW1lbnNpb25zQ2hhbmdlPW5ldyBjLkV2ZW50RW1pdHRlcixsLl9vblJlbmRlcj1uZXcgYy5FdmVudEVtaXR0ZXIsbC5fb25SZWZyZXNoUmVxdWVzdD1uZXcgYy5FdmVudEVtaXR0ZXIsbC5yZWdpc3Rlcih7ZGlzcG9zZTpmdW5jdGlvbigpe3JldHVybiBsLl9yZW5kZXJlci5kaXNwb3NlKCl9fSksbC5fcmVuZGVyRGVib3VuY2VyPW5ldyBhLlJlbmRlckRlYm91bmNlcigoZnVuY3Rpb24oZSx0KXtyZXR1cm4gbC5fcmVuZGVyUm93cyhlLHQpfSkpLGwucmVnaXN0ZXIobC5fcmVuZGVyRGVib3VuY2VyKSxsLl9zY3JlZW5EcHJNb25pdG9yPW5ldyB1LlNjcmVlbkRwck1vbml0b3IsbC5fc2NyZWVuRHByTW9uaXRvci5zZXRMaXN0ZW5lcigoZnVuY3Rpb24oKXtyZXR1cm4gbC5vbkRldmljZVBpeGVsUmF0aW9DaGFuZ2UoKX0pKSxsLnJlZ2lzdGVyKGwuX3NjcmVlbkRwck1vbml0b3IpLGwucmVnaXN0ZXIocy5vblJlc2l6ZSgoZnVuY3Rpb24oZSl7cmV0dXJuIGwuX2Z1bGxSZWZyZXNoKCl9KSkpLGwucmVnaXN0ZXIobi5vbk9wdGlvbkNoYW5nZSgoZnVuY3Rpb24oKXtyZXR1cm4gbC5fcmVuZGVyZXIub25PcHRpb25zQ2hhbmdlZCgpfSkpKSxsLnJlZ2lzdGVyKGwuX2NoYXJTaXplU2VydmljZS5vbkNoYXJTaXplQ2hhbmdlKChmdW5jdGlvbigpe3JldHVybiBsLm9uQ2hhclNpemVDaGFuZ2VkKCl9KSkpLGwuX3JlbmRlcmVyLm9uUmVxdWVzdFJlZHJhdygoZnVuY3Rpb24oZSl7cmV0dXJuIGwucmVmcmVzaFJvd3MoZS5zdGFydCxlLmVuZCwhMCl9KSksbC5yZWdpc3RlcigoMCxoLmFkZERpc3Bvc2FibGVEb21MaXN0ZW5lcikod2luZG93LCJyZXNpemUiLChmdW5jdGlvbigpe3JldHVybiBsLm9uRGV2aWNlUGl4ZWxSYXRpb0NoYW5nZSgpfSkpKSwiSW50ZXJzZWN0aW9uT2JzZXJ2ZXIiaW4gd2luZG93KXt2YXIgZj1uZXcgSW50ZXJzZWN0aW9uT2JzZXJ2ZXIoKGZ1bmN0aW9uKGUpe3JldHVybiBsLl9vbkludGVyc2VjdGlvbkNoYW5nZShlW2UubGVuZ3RoLTFdKX0pLHt0aHJlc2hvbGQ6MH0pO2Yub2JzZXJ2ZShpKSxsLnJlZ2lzdGVyKHtkaXNwb3NlOmZ1bmN0aW9uKCl7cmV0dXJuIGYuZGlzY29ubmVjdCgpfX0pfXJldHVybiBsfXJldHVybiBuKHQsZSksT2JqZWN0LmRlZmluZVByb3BlcnR5KHQucHJvdG90eXBlLCJvbkRpbWVuc2lvbnNDaGFuZ2UiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fb25EaW1lbnNpb25zQ2hhbmdlLmV2ZW50fSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLE9iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LnByb3RvdHlwZSwib25SZW5kZXJlZEJ1ZmZlckNoYW5nZSIse2dldDpmdW5jdGlvbigpe3JldHVybiB0aGlzLl9vblJlbmRlci5ldmVudH0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxPYmplY3QuZGVmaW5lUHJvcGVydHkodC5wcm90b3R5cGUsIm9uUmVmcmVzaFJlcXVlc3QiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fb25SZWZyZXNoUmVxdWVzdC5ldmVudH0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxPYmplY3QuZGVmaW5lUHJvcGVydHkodC5wcm90b3R5cGUsImRpbWVuc2lvbnMiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fcmVuZGVyZXIuZGltZW5zaW9uc30sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSx0LnByb3RvdHlwZS5fb25JbnRlcnNlY3Rpb25DaGFuZ2U9ZnVuY3Rpb24oZSl7dGhpcy5faXNQYXVzZWQ9dm9pZCAwPT09ZS5pc0ludGVyc2VjdGluZz8wPT09ZS5pbnRlcnNlY3Rpb25SYXRpbzohZS5pc0ludGVyc2VjdGluZyx0aGlzLl9pc1BhdXNlZHx8dGhpcy5fY2hhclNpemVTZXJ2aWNlLmhhc1ZhbGlkU2l6ZXx8dGhpcy5fY2hhclNpemVTZXJ2aWNlLm1lYXN1cmUoKSwhdGhpcy5faXNQYXVzZWQmJnRoaXMuX25lZWRzRnVsbFJlZnJlc2gmJih0aGlzLnJlZnJlc2hSb3dzKDAsdGhpcy5fcm93Q291bnQtMSksdGhpcy5fbmVlZHNGdWxsUmVmcmVzaD0hMSl9LHQucHJvdG90eXBlLnJlZnJlc2hSb3dzPWZ1bmN0aW9uKGUsdCxyKXt2b2lkIDA9PT1yJiYocj0hMSksdGhpcy5faXNQYXVzZWQ/dGhpcy5fbmVlZHNGdWxsUmVmcmVzaD0hMDoocnx8KHRoaXMuX2lzTmV4dFJlbmRlclJlZHJhd09ubHk9ITEpLHRoaXMuX3JlbmRlckRlYm91bmNlci5yZWZyZXNoKGUsdCx0aGlzLl9yb3dDb3VudCkpfSx0LnByb3RvdHlwZS5fcmVuZGVyUm93cz1mdW5jdGlvbihlLHQpe3RoaXMuX3JlbmRlcmVyLnJlbmRlclJvd3MoZSx0KSx0aGlzLl9uZWVkc1NlbGVjdGlvblJlZnJlc2gmJih0aGlzLl9yZW5kZXJlci5vblNlbGVjdGlvbkNoYW5nZWQodGhpcy5fc2VsZWN0aW9uU3RhdGUuc3RhcnQsdGhpcy5fc2VsZWN0aW9uU3RhdGUuZW5kLHRoaXMuX3NlbGVjdGlvblN0YXRlLmNvbHVtblNlbGVjdE1vZGUpLHRoaXMuX25lZWRzU2VsZWN0aW9uUmVmcmVzaD0hMSksdGhpcy5faXNOZXh0UmVuZGVyUmVkcmF3T25seXx8dGhpcy5fb25SZW5kZXIuZmlyZSh7c3RhcnQ6ZSxlbmQ6dH0pLHRoaXMuX2lzTmV4dFJlbmRlclJlZHJhd09ubHk9ITB9LHQucHJvdG90eXBlLnJlc2l6ZT1mdW5jdGlvbihlLHQpe3RoaXMuX3Jvd0NvdW50PXQsdGhpcy5fZmlyZU9uQ2FudmFzUmVzaXplKCl9LHQucHJvdG90eXBlLmNoYW5nZU9wdGlvbnM9ZnVuY3Rpb24oKXt0aGlzLl9yZW5kZXJlci5vbk9wdGlvbnNDaGFuZ2VkKCksdGhpcy5yZWZyZXNoUm93cygwLHRoaXMuX3Jvd0NvdW50LTEpLHRoaXMuX2ZpcmVPbkNhbnZhc1Jlc2l6ZSgpfSx0LnByb3RvdHlwZS5fZmlyZU9uQ2FudmFzUmVzaXplPWZ1bmN0aW9uKCl7dGhpcy5fcmVuZGVyZXIuZGltZW5zaW9ucy5jYW52YXNXaWR0aD09PXRoaXMuX2NhbnZhc1dpZHRoJiZ0aGlzLl9yZW5kZXJlci5kaW1lbnNpb25zLmNhbnZhc0hlaWdodD09PXRoaXMuX2NhbnZhc0hlaWdodHx8dGhpcy5fb25EaW1lbnNpb25zQ2hhbmdlLmZpcmUodGhpcy5fcmVuZGVyZXIuZGltZW5zaW9ucyl9LHQucHJvdG90eXBlLmRpc3Bvc2U9ZnVuY3Rpb24oKXtlLnByb3RvdHlwZS5kaXNwb3NlLmNhbGwodGhpcyl9LHQucHJvdG90eXBlLnNldFJlbmRlcmVyPWZ1bmN0aW9uKGUpe3ZhciB0PXRoaXM7dGhpcy5fcmVuZGVyZXIuZGlzcG9zZSgpLHRoaXMuX3JlbmRlcmVyPWUsdGhpcy5fcmVuZGVyZXIub25SZXF1ZXN0UmVkcmF3KChmdW5jdGlvbihlKXtyZXR1cm4gdC5yZWZyZXNoUm93cyhlLnN0YXJ0LGUuZW5kLCEwKX0pKSx0aGlzLl9uZWVkc1NlbGVjdGlvblJlZnJlc2g9ITAsdGhpcy5fZnVsbFJlZnJlc2goKX0sdC5wcm90b3R5cGUuX2Z1bGxSZWZyZXNoPWZ1bmN0aW9uKCl7dGhpcy5faXNQYXVzZWQ/dGhpcy5fbmVlZHNGdWxsUmVmcmVzaD0hMDp0aGlzLnJlZnJlc2hSb3dzKDAsdGhpcy5fcm93Q291bnQtMSl9LHQucHJvdG90eXBlLmNsZWFyVGV4dHVyZUF0bGFzPWZ1bmN0aW9uKCl7dmFyIGUsdDtudWxsPT09KHQ9bnVsbD09PShlPXRoaXMuX3JlbmRlcmVyKXx8dm9pZCAwPT09ZT92b2lkIDA6ZS5jbGVhclRleHR1cmVBdGxhcyl8fHZvaWQgMD09PXR8fHQuY2FsbChlKSx0aGlzLl9mdWxsUmVmcmVzaCgpfSx0LnByb3RvdHlwZS5zZXRDb2xvcnM9ZnVuY3Rpb24oZSl7dGhpcy5fcmVuZGVyZXIuc2V0Q29sb3JzKGUpLHRoaXMuX2Z1bGxSZWZyZXNoKCl9LHQucHJvdG90eXBlLm9uRGV2aWNlUGl4ZWxSYXRpb0NoYW5nZT1mdW5jdGlvbigpe3RoaXMuX2NoYXJTaXplU2VydmljZS5tZWFzdXJlKCksdGhpcy5fcmVuZGVyZXIub25EZXZpY2VQaXhlbFJhdGlvQ2hhbmdlKCksdGhpcy5yZWZyZXNoUm93cygwLHRoaXMuX3Jvd0NvdW50LTEpfSx0LnByb3RvdHlwZS5vblJlc2l6ZT1mdW5jdGlvbihlLHQpe3RoaXMuX3JlbmRlcmVyLm9uUmVzaXplKGUsdCksdGhpcy5fZnVsbFJlZnJlc2goKX0sdC5wcm90b3R5cGUub25DaGFyU2l6ZUNoYW5nZWQ9ZnVuY3Rpb24oKXt0aGlzLl9yZW5kZXJlci5vbkNoYXJTaXplQ2hhbmdlZCgpfSx0LnByb3RvdHlwZS5vbkJsdXI9ZnVuY3Rpb24oKXt0aGlzLl9yZW5kZXJlci5vbkJsdXIoKX0sdC5wcm90b3R5cGUub25Gb2N1cz1mdW5jdGlvbigpe3RoaXMuX3JlbmRlcmVyLm9uRm9jdXMoKX0sdC5wcm90b3R5cGUub25TZWxlY3Rpb25DaGFuZ2VkPWZ1bmN0aW9uKGUsdCxyKXt0aGlzLl9zZWxlY3Rpb25TdGF0ZS5zdGFydD1lLHRoaXMuX3NlbGVjdGlvblN0YXRlLmVuZD10LHRoaXMuX3NlbGVjdGlvblN0YXRlLmNvbHVtblNlbGVjdE1vZGU9cix0aGlzLl9yZW5kZXJlci5vblNlbGVjdGlvbkNoYW5nZWQoZSx0LHIpfSx0LnByb3RvdHlwZS5vbkN1cnNvck1vdmU9ZnVuY3Rpb24oKXt0aGlzLl9yZW5kZXJlci5vbkN1cnNvck1vdmUoKX0sdC5wcm90b3R5cGUuY2xlYXI9ZnVuY3Rpb24oKXt0aGlzLl9yZW5kZXJlci5jbGVhcigpfSxvKFtzKDMsZi5JT3B0aW9uc1NlcnZpY2UpLHMoNCxfLklDaGFyU2l6ZVNlcnZpY2UpLHMoNSxmLklCdWZmZXJTZXJ2aWNlKV0sdCl9KGwuRGlzcG9zYWJsZSk7dC5SZW5kZXJTZXJ2aWNlPWR9LDkzMTI6ZnVuY3Rpb24oZSx0LHIpe3ZhciBpLG49dGhpcyYmdGhpcy5fX2V4dGVuZHN8fChpPWZ1bmN0aW9uKGUsdCl7cmV0dXJuIGk9T2JqZWN0LnNldFByb3RvdHlwZU9mfHx7X19wcm90b19fOltdfWluc3RhbmNlb2YgQXJyYXkmJmZ1bmN0aW9uKGUsdCl7ZS5fX3Byb3RvX189dH18fGZ1bmN0aW9uKGUsdCl7Zm9yKHZhciByIGluIHQpT2JqZWN0LnByb3RvdHlwZS5oYXNPd25Qcm9wZXJ0eS5jYWxsKHQscikmJihlW3JdPXRbcl0pfSxpKGUsdCl9LGZ1bmN0aW9uKGUsdCl7aWYoImZ1bmN0aW9uIiE9dHlwZW9mIHQmJm51bGwhPT10KXRocm93IG5ldyBUeXBlRXJyb3IoIkNsYXNzIGV4dGVuZHMgdmFsdWUgIitTdHJpbmcodCkrIiBpcyBub3QgYSBjb25zdHJ1Y3RvciBvciBudWxsIik7ZnVuY3Rpb24gcigpe3RoaXMuY29uc3RydWN0b3I9ZX1pKGUsdCksZS5wcm90b3R5cGU9bnVsbD09PXQ/T2JqZWN0LmNyZWF0ZSh0KTooci5wcm90b3R5cGU9dC5wcm90b3R5cGUsbmV3IHIpfSksbz10aGlzJiZ0aGlzLl9fZGVjb3JhdGV8fGZ1bmN0aW9uKGUsdCxyLGkpe3ZhciBuLG89YXJndW1lbnRzLmxlbmd0aCxzPW88Mz90Om51bGw9PT1pP2k9T2JqZWN0LmdldE93blByb3BlcnR5RGVzY3JpcHRvcih0LHIpOmk7aWYoIm9iamVjdCI9PXR5cGVvZiBSZWZsZWN0JiYiZnVuY3Rpb24iPT10eXBlb2YgUmVmbGVjdC5kZWNvcmF0ZSlzPVJlZmxlY3QuZGVjb3JhdGUoZSx0LHIsaSk7ZWxzZSBmb3IodmFyIGE9ZS5sZW5ndGgtMTthPj0wO2EtLSkobj1lW2FdKSYmKHM9KG88Mz9uKHMpOm8+Mz9uKHQscixzKTpuKHQscikpfHxzKTtyZXR1cm4gbz4zJiZzJiZPYmplY3QuZGVmaW5lUHJvcGVydHkodCxyLHMpLHN9LHM9dGhpcyYmdGhpcy5fX3BhcmFtfHxmdW5jdGlvbihlLHQpe3JldHVybiBmdW5jdGlvbihyLGkpe3QocixpLGUpfX07T2JqZWN0LmRlZmluZVByb3BlcnR5KHQsIl9fZXNNb2R1bGUiLHt2YWx1ZTohMH0pLHQuU2VsZWN0aW9uU2VydmljZT12b2lkIDA7dmFyIGE9cig2MTE0KSxjPXIoNDU2KSxsPXIoNTExKSx1PXIoODQ2MCksaD1yKDQ3MjUpLGY9cigyNTg1KSxfPXIoOTgwNiksZD1yKDk1MDQpLHA9cig4NDQpLHY9cig0ODQxKSxnPVN0cmluZy5mcm9tQ2hhckNvZGUoMTYwKSx5PW5ldyBSZWdFeHAoZywiZyIpLG09ZnVuY3Rpb24oZSl7ZnVuY3Rpb24gdCh0LHIsaSxuLG8scyxhLGgpe3ZhciBmPWUuY2FsbCh0aGlzKXx8dGhpcztyZXR1cm4gZi5fZWxlbWVudD10LGYuX3NjcmVlbkVsZW1lbnQ9cixmLl9saW5raWZpZXI9aSxmLl9idWZmZXJTZXJ2aWNlPW4sZi5fY29yZVNlcnZpY2U9byxmLl9tb3VzZVNlcnZpY2U9cyxmLl9vcHRpb25zU2VydmljZT1hLGYuX3JlbmRlclNlcnZpY2U9aCxmLl9kcmFnU2Nyb2xsQW1vdW50PTAsZi5fZW5hYmxlZD0hMCxmLl93b3JrQ2VsbD1uZXcgbC5DZWxsRGF0YSxmLl9tb3VzZURvd25UaW1lU3RhbXA9MCxmLl9vbGRIYXNTZWxlY3Rpb249ITEsZi5fb2xkU2VsZWN0aW9uU3RhcnQ9dm9pZCAwLGYuX29sZFNlbGVjdGlvbkVuZD12b2lkIDAsZi5fb25MaW51eE1vdXNlU2VsZWN0aW9uPWYucmVnaXN0ZXIobmV3IHUuRXZlbnRFbWl0dGVyKSxmLl9vblJlZHJhd1JlcXVlc3Q9Zi5yZWdpc3RlcihuZXcgdS5FdmVudEVtaXR0ZXIpLGYuX29uU2VsZWN0aW9uQ2hhbmdlPWYucmVnaXN0ZXIobmV3IHUuRXZlbnRFbWl0dGVyKSxmLl9vblJlcXVlc3RTY3JvbGxMaW5lcz1mLnJlZ2lzdGVyKG5ldyB1LkV2ZW50RW1pdHRlciksZi5fbW91c2VNb3ZlTGlzdGVuZXI9ZnVuY3Rpb24oZSl7cmV0dXJuIGYuX29uTW91c2VNb3ZlKGUpfSxmLl9tb3VzZVVwTGlzdGVuZXI9ZnVuY3Rpb24oZSl7cmV0dXJuIGYuX29uTW91c2VVcChlKX0sZi5fY29yZVNlcnZpY2Uub25Vc2VySW5wdXQoKGZ1bmN0aW9uKCl7Zi5oYXNTZWxlY3Rpb24mJmYuY2xlYXJTZWxlY3Rpb24oKX0pKSxmLl90cmltTGlzdGVuZXI9Zi5fYnVmZmVyU2VydmljZS5idWZmZXIubGluZXMub25UcmltKChmdW5jdGlvbihlKXtyZXR1cm4gZi5fb25UcmltKGUpfSkpLGYucmVnaXN0ZXIoZi5fYnVmZmVyU2VydmljZS5idWZmZXJzLm9uQnVmZmVyQWN0aXZhdGUoKGZ1bmN0aW9uKGUpe3JldHVybiBmLl9vbkJ1ZmZlckFjdGl2YXRlKGUpfSkpKSxmLmVuYWJsZSgpLGYuX21vZGVsPW5ldyBjLlNlbGVjdGlvbk1vZGVsKGYuX2J1ZmZlclNlcnZpY2UpLGYuX2FjdGl2ZVNlbGVjdGlvbk1vZGU9MCxmfXJldHVybiBuKHQsZSksT2JqZWN0LmRlZmluZVByb3BlcnR5KHQucHJvdG90eXBlLCJvbkxpbnV4TW91c2VTZWxlY3Rpb24iLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fb25MaW51eE1vdXNlU2VsZWN0aW9uLmV2ZW50fSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLE9iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LnByb3RvdHlwZSwib25SZXF1ZXN0UmVkcmF3Iix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX29uUmVkcmF3UmVxdWVzdC5ldmVudH0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxPYmplY3QuZGVmaW5lUHJvcGVydHkodC5wcm90b3R5cGUsIm9uU2VsZWN0aW9uQ2hhbmdlIix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX29uU2VsZWN0aW9uQ2hhbmdlLmV2ZW50fSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLE9iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LnByb3RvdHlwZSwib25SZXF1ZXN0U2Nyb2xsTGluZXMiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fb25SZXF1ZXN0U2Nyb2xsTGluZXMuZXZlbnR9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksdC5wcm90b3R5cGUuZGlzcG9zZT1mdW5jdGlvbigpe3RoaXMuX3JlbW92ZU1vdXNlRG93bkxpc3RlbmVycygpfSx0LnByb3RvdHlwZS5yZXNldD1mdW5jdGlvbigpe3RoaXMuY2xlYXJTZWxlY3Rpb24oKX0sdC5wcm90b3R5cGUuZGlzYWJsZT1mdW5jdGlvbigpe3RoaXMuY2xlYXJTZWxlY3Rpb24oKSx0aGlzLl9lbmFibGVkPSExfSx0LnByb3RvdHlwZS5lbmFibGU9ZnVuY3Rpb24oKXt0aGlzLl9lbmFibGVkPSEwfSxPYmplY3QuZGVmaW5lUHJvcGVydHkodC5wcm90b3R5cGUsInNlbGVjdGlvblN0YXJ0Iix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX21vZGVsLmZpbmFsU2VsZWN0aW9uU3RhcnR9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KHQucHJvdG90eXBlLCJzZWxlY3Rpb25FbmQiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fbW9kZWwuZmluYWxTZWxlY3Rpb25FbmR9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KHQucHJvdG90eXBlLCJoYXNTZWxlY3Rpb24iLHtnZXQ6ZnVuY3Rpb24oKXt2YXIgZT10aGlzLl9tb2RlbC5maW5hbFNlbGVjdGlvblN0YXJ0LHQ9dGhpcy5fbW9kZWwuZmluYWxTZWxlY3Rpb25FbmQ7cmV0dXJuISghZXx8IXR8fGVbMF09PT10WzBdJiZlWzFdPT09dFsxXSl9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KHQucHJvdG90eXBlLCJzZWxlY3Rpb25UZXh0Iix7Z2V0OmZ1bmN0aW9uKCl7dmFyIGU9dGhpcy5fbW9kZWwuZmluYWxTZWxlY3Rpb25TdGFydCx0PXRoaXMuX21vZGVsLmZpbmFsU2VsZWN0aW9uRW5kO2lmKCFlfHwhdClyZXR1cm4iIjt2YXIgcj10aGlzLl9idWZmZXJTZXJ2aWNlLmJ1ZmZlcixpPVtdO2lmKDM9PT10aGlzLl9hY3RpdmVTZWxlY3Rpb25Nb2RlKXtpZihlWzBdPT09dFswXSlyZXR1cm4iIjtmb3IodmFyIG49ZVsxXTtuPD10WzFdO24rKyl7dmFyIG89ci50cmFuc2xhdGVCdWZmZXJMaW5lVG9TdHJpbmcobiwhMCxlWzBdLHRbMF0pO2kucHVzaChvKX19ZWxzZXt2YXIgcz1lWzFdPT09dFsxXT90WzBdOnZvaWQgMDtmb3IoaS5wdXNoKHIudHJhbnNsYXRlQnVmZmVyTGluZVRvU3RyaW5nKGVbMV0sITAsZVswXSxzKSksbj1lWzFdKzE7bjw9dFsxXS0xO24rKyl7dmFyIGM9ci5saW5lcy5nZXQobik7bz1yLnRyYW5zbGF0ZUJ1ZmZlckxpbmVUb1N0cmluZyhuLCEwKSwobnVsbD09Yz92b2lkIDA6Yy5pc1dyYXBwZWQpP2lbaS5sZW5ndGgtMV0rPW86aS5wdXNoKG8pfWVbMV0hPT10WzFdJiYoYz1yLmxpbmVzLmdldCh0WzFdKSxvPXIudHJhbnNsYXRlQnVmZmVyTGluZVRvU3RyaW5nKHRbMV0sITAsMCx0WzBdKSxjJiZjLmlzV3JhcHBlZD9pW2kubGVuZ3RoLTFdKz1vOmkucHVzaChvKSl9cmV0dXJuIGkubWFwKChmdW5jdGlvbihlKXtyZXR1cm4gZS5yZXBsYWNlKHksIiAiKX0pKS5qb2luKGEuaXNXaW5kb3dzPyJcclxuIjoiXG4iKX0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSx0LnByb3RvdHlwZS5jbGVhclNlbGVjdGlvbj1mdW5jdGlvbigpe3RoaXMuX21vZGVsLmNsZWFyU2VsZWN0aW9uKCksdGhpcy5fcmVtb3ZlTW91c2VEb3duTGlzdGVuZXJzKCksdGhpcy5yZWZyZXNoKCksdGhpcy5fb25TZWxlY3Rpb25DaGFuZ2UuZmlyZSgpfSx0LnByb3RvdHlwZS5yZWZyZXNoPWZ1bmN0aW9uKGUpe3ZhciB0PXRoaXM7dGhpcy5fcmVmcmVzaEFuaW1hdGlvbkZyYW1lfHwodGhpcy5fcmVmcmVzaEFuaW1hdGlvbkZyYW1lPXdpbmRvdy5yZXF1ZXN0QW5pbWF0aW9uRnJhbWUoKGZ1bmN0aW9uKCl7cmV0dXJuIHQuX3JlZnJlc2goKX0pKSksYS5pc0xpbnV4JiZlJiZ0aGlzLnNlbGVjdGlvblRleHQubGVuZ3RoJiZ0aGlzLl9vbkxpbnV4TW91c2VTZWxlY3Rpb24uZmlyZSh0aGlzLnNlbGVjdGlvblRleHQpfSx0LnByb3RvdHlwZS5fcmVmcmVzaD1mdW5jdGlvbigpe3RoaXMuX3JlZnJlc2hBbmltYXRpb25GcmFtZT12b2lkIDAsdGhpcy5fb25SZWRyYXdSZXF1ZXN0LmZpcmUoe3N0YXJ0OnRoaXMuX21vZGVsLmZpbmFsU2VsZWN0aW9uU3RhcnQsZW5kOnRoaXMuX21vZGVsLmZpbmFsU2VsZWN0aW9uRW5kLGNvbHVtblNlbGVjdE1vZGU6Mz09PXRoaXMuX2FjdGl2ZVNlbGVjdGlvbk1vZGV9KX0sdC5wcm90b3R5cGUuX2lzQ2xpY2tJblNlbGVjdGlvbj1mdW5jdGlvbihlKXt2YXIgdD10aGlzLl9nZXRNb3VzZUJ1ZmZlckNvb3JkcyhlKSxyPXRoaXMuX21vZGVsLmZpbmFsU2VsZWN0aW9uU3RhcnQsaT10aGlzLl9tb2RlbC5maW5hbFNlbGVjdGlvbkVuZDtyZXR1cm4hIShyJiZpJiZ0KSYmdGhpcy5fYXJlQ29vcmRzSW5TZWxlY3Rpb24odCxyLGkpfSx0LnByb3RvdHlwZS5fYXJlQ29vcmRzSW5TZWxlY3Rpb249ZnVuY3Rpb24oZSx0LHIpe3JldHVybiBlWzFdPnRbMV0mJmVbMV08clsxXXx8dFsxXT09PXJbMV0mJmVbMV09PT10WzFdJiZlWzBdPj10WzBdJiZlWzBdPHJbMF18fHRbMV08clsxXSYmZVsxXT09PXJbMV0mJmVbMF08clswXXx8dFsxXTxyWzFdJiZlWzFdPT09dFsxXSYmZVswXT49dFswXX0sdC5wcm90b3R5cGUuX3NlbGVjdFdvcmRBdEN1cnNvcj1mdW5jdGlvbihlLHQpe3ZhciByLGksbj1udWxsPT09KGk9bnVsbD09PShyPXRoaXMuX2xpbmtpZmllci5jdXJyZW50TGluayl8fHZvaWQgMD09PXI/dm9pZCAwOnIubGluayl8fHZvaWQgMD09PWk/dm9pZCAwOmkucmFuZ2U7aWYobilyZXR1cm4gdGhpcy5fbW9kZWwuc2VsZWN0aW9uU3RhcnQ9W24uc3RhcnQueC0xLG4uc3RhcnQueS0xXSx0aGlzLl9tb2RlbC5zZWxlY3Rpb25TdGFydExlbmd0aD0oMCx2LmdldFJhbmdlTGVuZ3RoKShuLHRoaXMuX2J1ZmZlclNlcnZpY2UuY29scyksdGhpcy5fbW9kZWwuc2VsZWN0aW9uRW5kPXZvaWQgMCwhMDt2YXIgbz10aGlzLl9nZXRNb3VzZUJ1ZmZlckNvb3JkcyhlKTtyZXR1cm4hIW8mJih0aGlzLl9zZWxlY3RXb3JkQXQobyx0KSx0aGlzLl9tb2RlbC5zZWxlY3Rpb25FbmQ9dm9pZCAwLCEwKX0sdC5wcm90b3R5cGUuc2VsZWN0QWxsPWZ1bmN0aW9uKCl7dGhpcy5fbW9kZWwuaXNTZWxlY3RBbGxBY3RpdmU9ITAsdGhpcy5yZWZyZXNoKCksdGhpcy5fb25TZWxlY3Rpb25DaGFuZ2UuZmlyZSgpfSx0LnByb3RvdHlwZS5zZWxlY3RMaW5lcz1mdW5jdGlvbihlLHQpe3RoaXMuX21vZGVsLmNsZWFyU2VsZWN0aW9uKCksZT1NYXRoLm1heChlLDApLHQ9TWF0aC5taW4odCx0aGlzLl9idWZmZXJTZXJ2aWNlLmJ1ZmZlci5saW5lcy5sZW5ndGgtMSksdGhpcy5fbW9kZWwuc2VsZWN0aW9uU3RhcnQ9WzAsZV0sdGhpcy5fbW9kZWwuc2VsZWN0aW9uRW5kPVt0aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHMsdF0sdGhpcy5yZWZyZXNoKCksdGhpcy5fb25TZWxlY3Rpb25DaGFuZ2UuZmlyZSgpfSx0LnByb3RvdHlwZS5fb25UcmltPWZ1bmN0aW9uKGUpe3RoaXMuX21vZGVsLm9uVHJpbShlKSYmdGhpcy5yZWZyZXNoKCl9LHQucHJvdG90eXBlLl9nZXRNb3VzZUJ1ZmZlckNvb3Jkcz1mdW5jdGlvbihlKXt2YXIgdD10aGlzLl9tb3VzZVNlcnZpY2UuZ2V0Q29vcmRzKGUsdGhpcy5fc2NyZWVuRWxlbWVudCx0aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHMsdGhpcy5fYnVmZmVyU2VydmljZS5yb3dzLCEwKTtpZih0KXJldHVybiB0WzBdLS0sdFsxXS0tLHRbMV0rPXRoaXMuX2J1ZmZlclNlcnZpY2UuYnVmZmVyLnlkaXNwLHR9LHQucHJvdG90eXBlLl9nZXRNb3VzZUV2ZW50U2Nyb2xsQW1vdW50PWZ1bmN0aW9uKGUpe3ZhciB0PSgwLF8uZ2V0Q29vcmRzUmVsYXRpdmVUb0VsZW1lbnQpKGUsdGhpcy5fc2NyZWVuRWxlbWVudClbMV0scj10aGlzLl9yZW5kZXJTZXJ2aWNlLmRpbWVuc2lvbnMuY2FudmFzSGVpZ2h0O3JldHVybiB0Pj0wJiZ0PD1yPzA6KHQ+ciYmKHQtPXIpLHQ9TWF0aC5taW4oTWF0aC5tYXgodCwtNTApLDUwKSwodC89NTApL01hdGguYWJzKHQpK01hdGgucm91bmQoMTQqdCkpfSx0LnByb3RvdHlwZS5zaG91bGRGb3JjZVNlbGVjdGlvbj1mdW5jdGlvbihlKXtyZXR1cm4gYS5pc01hYz9lLmFsdEtleSYmdGhpcy5fb3B0aW9uc1NlcnZpY2Uub3B0aW9ucy5tYWNPcHRpb25DbGlja0ZvcmNlc1NlbGVjdGlvbjplLnNoaWZ0S2V5fSx0LnByb3RvdHlwZS5vbk1vdXNlRG93bj1mdW5jdGlvbihlKXtpZih0aGlzLl9tb3VzZURvd25UaW1lU3RhbXA9ZS50aW1lU3RhbXAsKDIhPT1lLmJ1dHRvbnx8IXRoaXMuaGFzU2VsZWN0aW9uKSYmMD09PWUuYnV0dG9uKXtpZighdGhpcy5fZW5hYmxlZCl7aWYoIXRoaXMuc2hvdWxkRm9yY2VTZWxlY3Rpb24oZSkpcmV0dXJuO2Uuc3RvcFByb3BhZ2F0aW9uKCl9ZS5wcmV2ZW50RGVmYXVsdCgpLHRoaXMuX2RyYWdTY3JvbGxBbW91bnQ9MCx0aGlzLl9lbmFibGVkJiZlLnNoaWZ0S2V5P3RoaXMuX29uSW5jcmVtZW50YWxDbGljayhlKToxPT09ZS5kZXRhaWw/dGhpcy5fb25TaW5nbGVDbGljayhlKToyPT09ZS5kZXRhaWw/dGhpcy5fb25Eb3VibGVDbGljayhlKTozPT09ZS5kZXRhaWwmJnRoaXMuX29uVHJpcGxlQ2xpY2soZSksdGhpcy5fYWRkTW91c2VEb3duTGlzdGVuZXJzKCksdGhpcy5yZWZyZXNoKCEwKX19LHQucHJvdG90eXBlLl9hZGRNb3VzZURvd25MaXN0ZW5lcnM9ZnVuY3Rpb24oKXt2YXIgZT10aGlzO3RoaXMuX3NjcmVlbkVsZW1lbnQub3duZXJEb2N1bWVudCYmKHRoaXMuX3NjcmVlbkVsZW1lbnQub3duZXJEb2N1bWVudC5hZGRFdmVudExpc3RlbmVyKCJtb3VzZW1vdmUiLHRoaXMuX21vdXNlTW92ZUxpc3RlbmVyKSx0aGlzLl9zY3JlZW5FbGVtZW50Lm93bmVyRG9jdW1lbnQuYWRkRXZlbnRMaXN0ZW5lcigibW91c2V1cCIsdGhpcy5fbW91c2VVcExpc3RlbmVyKSksdGhpcy5fZHJhZ1Njcm9sbEludGVydmFsVGltZXI9d2luZG93LnNldEludGVydmFsKChmdW5jdGlvbigpe3JldHVybiBlLl9kcmFnU2Nyb2xsKCl9KSw1MCl9LHQucHJvdG90eXBlLl9yZW1vdmVNb3VzZURvd25MaXN0ZW5lcnM9ZnVuY3Rpb24oKXt0aGlzLl9zY3JlZW5FbGVtZW50Lm93bmVyRG9jdW1lbnQmJih0aGlzLl9zY3JlZW5FbGVtZW50Lm93bmVyRG9jdW1lbnQucmVtb3ZlRXZlbnRMaXN0ZW5lcigibW91c2Vtb3ZlIix0aGlzLl9tb3VzZU1vdmVMaXN0ZW5lciksdGhpcy5fc2NyZWVuRWxlbWVudC5vd25lckRvY3VtZW50LnJlbW92ZUV2ZW50TGlzdGVuZXIoIm1vdXNldXAiLHRoaXMuX21vdXNlVXBMaXN0ZW5lcikpLGNsZWFySW50ZXJ2YWwodGhpcy5fZHJhZ1Njcm9sbEludGVydmFsVGltZXIpLHRoaXMuX2RyYWdTY3JvbGxJbnRlcnZhbFRpbWVyPXZvaWQgMH0sdC5wcm90b3R5cGUuX29uSW5jcmVtZW50YWxDbGljaz1mdW5jdGlvbihlKXt0aGlzLl9tb2RlbC5zZWxlY3Rpb25TdGFydCYmKHRoaXMuX21vZGVsLnNlbGVjdGlvbkVuZD10aGlzLl9nZXRNb3VzZUJ1ZmZlckNvb3JkcyhlKSl9LHQucHJvdG90eXBlLl9vblNpbmdsZUNsaWNrPWZ1bmN0aW9uKGUpe2lmKHRoaXMuX21vZGVsLnNlbGVjdGlvblN0YXJ0TGVuZ3RoPTAsdGhpcy5fbW9kZWwuaXNTZWxlY3RBbGxBY3RpdmU9ITEsdGhpcy5fYWN0aXZlU2VsZWN0aW9uTW9kZT10aGlzLnNob3VsZENvbHVtblNlbGVjdChlKT8zOjAsdGhpcy5fbW9kZWwuc2VsZWN0aW9uU3RhcnQ9dGhpcy5fZ2V0TW91c2VCdWZmZXJDb29yZHMoZSksdGhpcy5fbW9kZWwuc2VsZWN0aW9uU3RhcnQpe3RoaXMuX21vZGVsLnNlbGVjdGlvbkVuZD12b2lkIDA7dmFyIHQ9dGhpcy5fYnVmZmVyU2VydmljZS5idWZmZXIubGluZXMuZ2V0KHRoaXMuX21vZGVsLnNlbGVjdGlvblN0YXJ0WzFdKTt0JiZ0Lmxlbmd0aCE9PXRoaXMuX21vZGVsLnNlbGVjdGlvblN0YXJ0WzBdJiYwPT09dC5oYXNXaWR0aCh0aGlzLl9tb2RlbC5zZWxlY3Rpb25TdGFydFswXSkmJnRoaXMuX21vZGVsLnNlbGVjdGlvblN0YXJ0WzBdKyt9fSx0LnByb3RvdHlwZS5fb25Eb3VibGVDbGljaz1mdW5jdGlvbihlKXt0aGlzLl9zZWxlY3RXb3JkQXRDdXJzb3IoZSwhMCkmJih0aGlzLl9hY3RpdmVTZWxlY3Rpb25Nb2RlPTEpfSx0LnByb3RvdHlwZS5fb25UcmlwbGVDbGljaz1mdW5jdGlvbihlKXt2YXIgdD10aGlzLl9nZXRNb3VzZUJ1ZmZlckNvb3JkcyhlKTt0JiYodGhpcy5fYWN0aXZlU2VsZWN0aW9uTW9kZT0yLHRoaXMuX3NlbGVjdExpbmVBdCh0WzFdKSl9LHQucHJvdG90eXBlLnNob3VsZENvbHVtblNlbGVjdD1mdW5jdGlvbihlKXtyZXR1cm4gZS5hbHRLZXkmJiEoYS5pc01hYyYmdGhpcy5fb3B0aW9uc1NlcnZpY2Uub3B0aW9ucy5tYWNPcHRpb25DbGlja0ZvcmNlc1NlbGVjdGlvbil9LHQucHJvdG90eXBlLl9vbk1vdXNlTW92ZT1mdW5jdGlvbihlKXtpZihlLnN0b3BJbW1lZGlhdGVQcm9wYWdhdGlvbigpLHRoaXMuX21vZGVsLnNlbGVjdGlvblN0YXJ0KXt2YXIgdD10aGlzLl9tb2RlbC5zZWxlY3Rpb25FbmQ/W3RoaXMuX21vZGVsLnNlbGVjdGlvbkVuZFswXSx0aGlzLl9tb2RlbC5zZWxlY3Rpb25FbmRbMV1dOm51bGw7aWYodGhpcy5fbW9kZWwuc2VsZWN0aW9uRW5kPXRoaXMuX2dldE1vdXNlQnVmZmVyQ29vcmRzKGUpLHRoaXMuX21vZGVsLnNlbGVjdGlvbkVuZCl7Mj09PXRoaXMuX2FjdGl2ZVNlbGVjdGlvbk1vZGU/dGhpcy5fbW9kZWwuc2VsZWN0aW9uRW5kWzFdPHRoaXMuX21vZGVsLnNlbGVjdGlvblN0YXJ0WzFdP3RoaXMuX21vZGVsLnNlbGVjdGlvbkVuZFswXT0wOnRoaXMuX21vZGVsLnNlbGVjdGlvbkVuZFswXT10aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHM6MT09PXRoaXMuX2FjdGl2ZVNlbGVjdGlvbk1vZGUmJnRoaXMuX3NlbGVjdFRvV29yZEF0KHRoaXMuX21vZGVsLnNlbGVjdGlvbkVuZCksdGhpcy5fZHJhZ1Njcm9sbEFtb3VudD10aGlzLl9nZXRNb3VzZUV2ZW50U2Nyb2xsQW1vdW50KGUpLDMhPT10aGlzLl9hY3RpdmVTZWxlY3Rpb25Nb2RlJiYodGhpcy5fZHJhZ1Njcm9sbEFtb3VudD4wP3RoaXMuX21vZGVsLnNlbGVjdGlvbkVuZFswXT10aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHM6dGhpcy5fZHJhZ1Njcm9sbEFtb3VudDwwJiYodGhpcy5fbW9kZWwuc2VsZWN0aW9uRW5kWzBdPTApKTt2YXIgcj10aGlzLl9idWZmZXJTZXJ2aWNlLmJ1ZmZlcjtpZih0aGlzLl9tb2RlbC5zZWxlY3Rpb25FbmRbMV08ci5saW5lcy5sZW5ndGgpe3ZhciBpPXIubGluZXMuZ2V0KHRoaXMuX21vZGVsLnNlbGVjdGlvbkVuZFsxXSk7aSYmMD09PWkuaGFzV2lkdGgodGhpcy5fbW9kZWwuc2VsZWN0aW9uRW5kWzBdKSYmdGhpcy5fbW9kZWwuc2VsZWN0aW9uRW5kWzBdKyt9dCYmdFswXT09PXRoaXMuX21vZGVsLnNlbGVjdGlvbkVuZFswXSYmdFsxXT09PXRoaXMuX21vZGVsLnNlbGVjdGlvbkVuZFsxXXx8dGhpcy5yZWZyZXNoKCEwKX1lbHNlIHRoaXMucmVmcmVzaCghMCl9fSx0LnByb3RvdHlwZS5fZHJhZ1Njcm9sbD1mdW5jdGlvbigpe2lmKHRoaXMuX21vZGVsLnNlbGVjdGlvbkVuZCYmdGhpcy5fbW9kZWwuc2VsZWN0aW9uU3RhcnQmJnRoaXMuX2RyYWdTY3JvbGxBbW91bnQpe3RoaXMuX29uUmVxdWVzdFNjcm9sbExpbmVzLmZpcmUoe2Ftb3VudDp0aGlzLl9kcmFnU2Nyb2xsQW1vdW50LHN1cHByZXNzU2Nyb2xsRXZlbnQ6ITF9KTt2YXIgZT10aGlzLl9idWZmZXJTZXJ2aWNlLmJ1ZmZlcjt0aGlzLl9kcmFnU2Nyb2xsQW1vdW50PjA/KDMhPT10aGlzLl9hY3RpdmVTZWxlY3Rpb25Nb2RlJiYodGhpcy5fbW9kZWwuc2VsZWN0aW9uRW5kWzBdPXRoaXMuX2J1ZmZlclNlcnZpY2UuY29scyksdGhpcy5fbW9kZWwuc2VsZWN0aW9uRW5kWzFdPU1hdGgubWluKGUueWRpc3ArdGhpcy5fYnVmZmVyU2VydmljZS5yb3dzLGUubGluZXMubGVuZ3RoLTEpKTooMyE9PXRoaXMuX2FjdGl2ZVNlbGVjdGlvbk1vZGUmJih0aGlzLl9tb2RlbC5zZWxlY3Rpb25FbmRbMF09MCksdGhpcy5fbW9kZWwuc2VsZWN0aW9uRW5kWzFdPWUueWRpc3ApLHRoaXMucmVmcmVzaCgpfX0sdC5wcm90b3R5cGUuX29uTW91c2VVcD1mdW5jdGlvbihlKXt2YXIgdD1lLnRpbWVTdGFtcC10aGlzLl9tb3VzZURvd25UaW1lU3RhbXA7aWYodGhpcy5fcmVtb3ZlTW91c2VEb3duTGlzdGVuZXJzKCksdGhpcy5zZWxlY3Rpb25UZXh0Lmxlbmd0aDw9MSYmdDw1MDAmJmUuYWx0S2V5JiZ0aGlzLl9vcHRpb25zU2VydmljZS5nZXRPcHRpb24oImFsdENsaWNrTW92ZXNDdXJzb3IiKSl7aWYodGhpcy5fYnVmZmVyU2VydmljZS5idWZmZXIueWJhc2U9PT10aGlzLl9idWZmZXJTZXJ2aWNlLmJ1ZmZlci55ZGlzcCl7dmFyIHI9dGhpcy5fbW91c2VTZXJ2aWNlLmdldENvb3JkcyhlLHRoaXMuX2VsZW1lbnQsdGhpcy5fYnVmZmVyU2VydmljZS5jb2xzLHRoaXMuX2J1ZmZlclNlcnZpY2Uucm93cywhMSk7aWYociYmdm9pZCAwIT09clswXSYmdm9pZCAwIT09clsxXSl7dmFyIGk9KDAsZC5tb3ZlVG9DZWxsU2VxdWVuY2UpKHJbMF0tMSxyWzFdLTEsdGhpcy5fYnVmZmVyU2VydmljZSx0aGlzLl9jb3JlU2VydmljZS5kZWNQcml2YXRlTW9kZXMuYXBwbGljYXRpb25DdXJzb3JLZXlzKTt0aGlzLl9jb3JlU2VydmljZS50cmlnZ2VyRGF0YUV2ZW50KGksITApfX19ZWxzZSB0aGlzLl9maXJlRXZlbnRJZlNlbGVjdGlvbkNoYW5nZWQoKX0sdC5wcm90b3R5cGUuX2ZpcmVFdmVudElmU2VsZWN0aW9uQ2hhbmdlZD1mdW5jdGlvbigpe3ZhciBlPXRoaXMuX21vZGVsLmZpbmFsU2VsZWN0aW9uU3RhcnQsdD10aGlzLl9tb2RlbC5maW5hbFNlbGVjdGlvbkVuZCxyPSEoIWV8fCF0fHxlWzBdPT09dFswXSYmZVsxXT09PXRbMV0pO3I/ZSYmdCYmKHRoaXMuX29sZFNlbGVjdGlvblN0YXJ0JiZ0aGlzLl9vbGRTZWxlY3Rpb25FbmQmJmVbMF09PT10aGlzLl9vbGRTZWxlY3Rpb25TdGFydFswXSYmZVsxXT09PXRoaXMuX29sZFNlbGVjdGlvblN0YXJ0WzFdJiZ0WzBdPT09dGhpcy5fb2xkU2VsZWN0aW9uRW5kWzBdJiZ0WzFdPT09dGhpcy5fb2xkU2VsZWN0aW9uRW5kWzFdfHx0aGlzLl9maXJlT25TZWxlY3Rpb25DaGFuZ2UoZSx0LHIpKTp0aGlzLl9vbGRIYXNTZWxlY3Rpb24mJnRoaXMuX2ZpcmVPblNlbGVjdGlvbkNoYW5nZShlLHQscil9LHQucHJvdG90eXBlLl9maXJlT25TZWxlY3Rpb25DaGFuZ2U9ZnVuY3Rpb24oZSx0LHIpe3RoaXMuX29sZFNlbGVjdGlvblN0YXJ0PWUsdGhpcy5fb2xkU2VsZWN0aW9uRW5kPXQsdGhpcy5fb2xkSGFzU2VsZWN0aW9uPXIsdGhpcy5fb25TZWxlY3Rpb25DaGFuZ2UuZmlyZSgpfSx0LnByb3RvdHlwZS5fb25CdWZmZXJBY3RpdmF0ZT1mdW5jdGlvbihlKXt2YXIgdD10aGlzO3RoaXMuY2xlYXJTZWxlY3Rpb24oKSx0aGlzLl90cmltTGlzdGVuZXIuZGlzcG9zZSgpLHRoaXMuX3RyaW1MaXN0ZW5lcj1lLmFjdGl2ZUJ1ZmZlci5saW5lcy5vblRyaW0oKGZ1bmN0aW9uKGUpe3JldHVybiB0Ll9vblRyaW0oZSl9KSl9LHQucHJvdG90eXBlLl9jb252ZXJ0Vmlld3BvcnRDb2xUb0NoYXJhY3RlckluZGV4PWZ1bmN0aW9uKGUsdCl7Zm9yKHZhciByPXRbMF0saT0wO3RbMF0+PWk7aSsrKXt2YXIgbj1lLmxvYWRDZWxsKGksdGhpcy5fd29ya0NlbGwpLmdldENoYXJzKCkubGVuZ3RoOzA9PT10aGlzLl93b3JrQ2VsbC5nZXRXaWR0aCgpP3ItLTpuPjEmJnRbMF0hPT1pJiYocis9bi0xKX1yZXR1cm4gcn0sdC5wcm90b3R5cGUuc2V0U2VsZWN0aW9uPWZ1bmN0aW9uKGUsdCxyKXt0aGlzLl9tb2RlbC5jbGVhclNlbGVjdGlvbigpLHRoaXMuX3JlbW92ZU1vdXNlRG93bkxpc3RlbmVycygpLHRoaXMuX21vZGVsLnNlbGVjdGlvblN0YXJ0PVtlLHRdLHRoaXMuX21vZGVsLnNlbGVjdGlvblN0YXJ0TGVuZ3RoPXIsdGhpcy5yZWZyZXNoKCl9LHQucHJvdG90eXBlLnJpZ2h0Q2xpY2tTZWxlY3Q9ZnVuY3Rpb24oZSl7dGhpcy5faXNDbGlja0luU2VsZWN0aW9uKGUpfHwodGhpcy5fc2VsZWN0V29yZEF0Q3Vyc29yKGUsITEpJiZ0aGlzLnJlZnJlc2goITApLHRoaXMuX2ZpcmVFdmVudElmU2VsZWN0aW9uQ2hhbmdlZCgpKX0sdC5wcm90b3R5cGUuX2dldFdvcmRBdD1mdW5jdGlvbihlLHQscixpKXtpZih2b2lkIDA9PT1yJiYocj0hMCksdm9pZCAwPT09aSYmKGk9ITApLCEoZVswXT49dGhpcy5fYnVmZmVyU2VydmljZS5jb2xzKSl7dmFyIG49dGhpcy5fYnVmZmVyU2VydmljZS5idWZmZXIsbz1uLmxpbmVzLmdldChlWzFdKTtpZihvKXt2YXIgcz1uLnRyYW5zbGF0ZUJ1ZmZlckxpbmVUb1N0cmluZyhlWzFdLCExKSxhPXRoaXMuX2NvbnZlcnRWaWV3cG9ydENvbFRvQ2hhcmFjdGVySW5kZXgobyxlKSxjPWEsbD1lWzBdLWEsdT0wLGg9MCxmPTAsXz0wO2lmKCIgIj09PXMuY2hhckF0KGEpKXtmb3IoO2E+MCYmIiAiPT09cy5jaGFyQXQoYS0xKTspYS0tO2Zvcig7YzxzLmxlbmd0aCYmIiAiPT09cy5jaGFyQXQoYysxKTspYysrfWVsc2V7dmFyIGQ9ZVswXSxwPWVbMF07MD09PW8uZ2V0V2lkdGgoZCkmJih1KyssZC0tKSwyPT09by5nZXRXaWR0aChwKSYmKGgrKyxwKyspO3ZhciB2PW8uZ2V0U3RyaW5nKHApLmxlbmd0aDtmb3Iodj4xJiYoXys9di0xLGMrPXYtMSk7ZD4wJiZhPjAmJiF0aGlzLl9pc0NoYXJXb3JkU2VwYXJhdG9yKG8ubG9hZENlbGwoZC0xLHRoaXMuX3dvcmtDZWxsKSk7KXtvLmxvYWRDZWxsKGQtMSx0aGlzLl93b3JrQ2VsbCk7dmFyIGc9dGhpcy5fd29ya0NlbGwuZ2V0Q2hhcnMoKS5sZW5ndGg7MD09PXRoaXMuX3dvcmtDZWxsLmdldFdpZHRoKCk/KHUrKyxkLS0pOmc+MSYmKGYrPWctMSxhLT1nLTEpLGEtLSxkLS19Zm9yKDtwPG8ubGVuZ3RoJiZjKzE8cy5sZW5ndGgmJiF0aGlzLl9pc0NoYXJXb3JkU2VwYXJhdG9yKG8ubG9hZENlbGwocCsxLHRoaXMuX3dvcmtDZWxsKSk7KXtvLmxvYWRDZWxsKHArMSx0aGlzLl93b3JrQ2VsbCk7dmFyIHk9dGhpcy5fd29ya0NlbGwuZ2V0Q2hhcnMoKS5sZW5ndGg7Mj09PXRoaXMuX3dvcmtDZWxsLmdldFdpZHRoKCk/KGgrKyxwKyspOnk+MSYmKF8rPXktMSxjKz15LTEpLGMrKyxwKyt9fWMrKzt2YXIgbT1hK2wtdStmLGI9TWF0aC5taW4odGhpcy5fYnVmZmVyU2VydmljZS5jb2xzLGMtYSt1K2gtZi1fKTtpZih0fHwiIiE9PXMuc2xpY2UoYSxjKS50cmltKCkpe2lmKHImJjA9PT1tJiYzMiE9PW8uZ2V0Q29kZVBvaW50KDApKXt2YXIgUz1uLmxpbmVzLmdldChlWzFdLTEpO2lmKFMmJm8uaXNXcmFwcGVkJiYzMiE9PVMuZ2V0Q29kZVBvaW50KHRoaXMuX2J1ZmZlclNlcnZpY2UuY29scy0xKSl7dmFyIEM9dGhpcy5fZ2V0V29yZEF0KFt0aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHMtMSxlWzFdLTFdLCExLCEwLCExKTtpZihDKXt2YXIgdz10aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHMtQy5zdGFydDttLT13LGIrPXd9fX1pZihpJiZtK2I9PT10aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHMmJjMyIT09by5nZXRDb2RlUG9pbnQodGhpcy5fYnVmZmVyU2VydmljZS5jb2xzLTEpKXt2YXIgTD1uLmxpbmVzLmdldChlWzFdKzEpO2lmKChudWxsPT1MP3ZvaWQgMDpMLmlzV3JhcHBlZCkmJjMyIT09TC5nZXRDb2RlUG9pbnQoMCkpe3ZhciBFPXRoaXMuX2dldFdvcmRBdChbMCxlWzFdKzFdLCExLCExLCEwKTtFJiYoYis9RS5sZW5ndGgpfX1yZXR1cm57c3RhcnQ6bSxsZW5ndGg6Yn19fX19LHQucHJvdG90eXBlLl9zZWxlY3RXb3JkQXQ9ZnVuY3Rpb24oZSx0KXt2YXIgcj10aGlzLl9nZXRXb3JkQXQoZSx0KTtpZihyKXtmb3IoO3Iuc3RhcnQ8MDspci5zdGFydCs9dGhpcy5fYnVmZmVyU2VydmljZS5jb2xzLGVbMV0tLTt0aGlzLl9tb2RlbC5zZWxlY3Rpb25TdGFydD1bci5zdGFydCxlWzFdXSx0aGlzLl9tb2RlbC5zZWxlY3Rpb25TdGFydExlbmd0aD1yLmxlbmd0aH19LHQucHJvdG90eXBlLl9zZWxlY3RUb1dvcmRBdD1mdW5jdGlvbihlKXt2YXIgdD10aGlzLl9nZXRXb3JkQXQoZSwhMCk7aWYodCl7Zm9yKHZhciByPWVbMV07dC5zdGFydDwwOyl0LnN0YXJ0Kz10aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHMsci0tO2lmKCF0aGlzLl9tb2RlbC5hcmVTZWxlY3Rpb25WYWx1ZXNSZXZlcnNlZCgpKWZvcig7dC5zdGFydCt0Lmxlbmd0aD50aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHM7KXQubGVuZ3RoLT10aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHMscisrO3RoaXMuX21vZGVsLnNlbGVjdGlvbkVuZD1bdGhpcy5fbW9kZWwuYXJlU2VsZWN0aW9uVmFsdWVzUmV2ZXJzZWQoKT90LnN0YXJ0OnQuc3RhcnQrdC5sZW5ndGgscl19fSx0LnByb3RvdHlwZS5faXNDaGFyV29yZFNlcGFyYXRvcj1mdW5jdGlvbihlKXtyZXR1cm4gMCE9PWUuZ2V0V2lkdGgoKSYmdGhpcy5fb3B0aW9uc1NlcnZpY2Uub3B0aW9ucy53b3JkU2VwYXJhdG9yLmluZGV4T2YoZS5nZXRDaGFycygpKT49MH0sdC5wcm90b3R5cGUuX3NlbGVjdExpbmVBdD1mdW5jdGlvbihlKXt2YXIgdD10aGlzLl9idWZmZXJTZXJ2aWNlLmJ1ZmZlci5nZXRXcmFwcGVkUmFuZ2VGb3JMaW5lKGUpO3RoaXMuX21vZGVsLnNlbGVjdGlvblN0YXJ0PVswLHQuZmlyc3RdLHRoaXMuX21vZGVsLnNlbGVjdGlvbkVuZD1bdGhpcy5fYnVmZmVyU2VydmljZS5jb2xzLHQubGFzdF0sdGhpcy5fbW9kZWwuc2VsZWN0aW9uU3RhcnRMZW5ndGg9MH0sbyhbcygzLGYuSUJ1ZmZlclNlcnZpY2UpLHMoNCxmLklDb3JlU2VydmljZSkscyg1LGguSU1vdXNlU2VydmljZSkscyg2LGYuSU9wdGlvbnNTZXJ2aWNlKSxzKDcsaC5JUmVuZGVyU2VydmljZSldLHQpfShwLkRpc3Bvc2FibGUpO3QuU2VsZWN0aW9uU2VydmljZT1tfSw0NzI1OihlLHQscik9PntPYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC5JQ2hhcmFjdGVySm9pbmVyU2VydmljZT10LklTb3VuZFNlcnZpY2U9dC5JU2VsZWN0aW9uU2VydmljZT10LklSZW5kZXJTZXJ2aWNlPXQuSU1vdXNlU2VydmljZT10LklDb3JlQnJvd3NlclNlcnZpY2U9dC5JQ2hhclNpemVTZXJ2aWNlPXZvaWQgMDt2YXIgaT1yKDgzNDMpO3QuSUNoYXJTaXplU2VydmljZT0oMCxpLmNyZWF0ZURlY29yYXRvcikoIkNoYXJTaXplU2VydmljZSIpLHQuSUNvcmVCcm93c2VyU2VydmljZT0oMCxpLmNyZWF0ZURlY29yYXRvcikoIkNvcmVCcm93c2VyU2VydmljZSIpLHQuSU1vdXNlU2VydmljZT0oMCxpLmNyZWF0ZURlY29yYXRvcikoIk1vdXNlU2VydmljZSIpLHQuSVJlbmRlclNlcnZpY2U9KDAsaS5jcmVhdGVEZWNvcmF0b3IpKCJSZW5kZXJTZXJ2aWNlIiksdC5JU2VsZWN0aW9uU2VydmljZT0oMCxpLmNyZWF0ZURlY29yYXRvcikoIlNlbGVjdGlvblNlcnZpY2UiKSx0LklTb3VuZFNlcnZpY2U9KDAsaS5jcmVhdGVEZWNvcmF0b3IpKCJTb3VuZFNlcnZpY2UiKSx0LklDaGFyYWN0ZXJKb2luZXJTZXJ2aWNlPSgwLGkuY3JlYXRlRGVjb3JhdG9yKSgiQ2hhcmFjdGVySm9pbmVyU2VydmljZSIpfSwzNTc6ZnVuY3Rpb24oZSx0LHIpe3ZhciBpPXRoaXMmJnRoaXMuX19kZWNvcmF0ZXx8ZnVuY3Rpb24oZSx0LHIsaSl7dmFyIG4sbz1hcmd1bWVudHMubGVuZ3RoLHM9bzwzP3Q6bnVsbD09PWk/aT1PYmplY3QuZ2V0T3duUHJvcGVydHlEZXNjcmlwdG9yKHQscik6aTtpZigib2JqZWN0Ij09dHlwZW9mIFJlZmxlY3QmJiJmdW5jdGlvbiI9PXR5cGVvZiBSZWZsZWN0LmRlY29yYXRlKXM9UmVmbGVjdC5kZWNvcmF0ZShlLHQscixpKTtlbHNlIGZvcih2YXIgYT1lLmxlbmd0aC0xO2E+PTA7YS0tKShuPWVbYV0pJiYocz0obzwzP24ocyk6bz4zP24odCxyLHMpOm4odCxyKSl8fHMpO3JldHVybiBvPjMmJnMmJk9iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LHIscyksc30sbj10aGlzJiZ0aGlzLl9fcGFyYW18fGZ1bmN0aW9uKGUsdCl7cmV0dXJuIGZ1bmN0aW9uKHIsaSl7dChyLGksZSl9fTtPYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC5Tb3VuZFNlcnZpY2U9dm9pZCAwO3ZhciBvPXIoMjU4NSkscz1mdW5jdGlvbigpe2Z1bmN0aW9uIGUoZSl7dGhpcy5fb3B0aW9uc1NlcnZpY2U9ZX1yZXR1cm4gT2JqZWN0LmRlZmluZVByb3BlcnR5KGUsImF1ZGlvQ29udGV4dCIse2dldDpmdW5jdGlvbigpe2lmKCFlLl9hdWRpb0NvbnRleHQpe3ZhciB0PXdpbmRvdy5BdWRpb0NvbnRleHR8fHdpbmRvdy53ZWJraXRBdWRpb0NvbnRleHQ7aWYoIXQpcmV0dXJuIGNvbnNvbGUud2FybigiV2ViIEF1ZGlvIEFQSSBpcyBub3Qgc3VwcG9ydGVkIGJ5IHRoaXMgYnJvd3Nlci4gQ29uc2lkZXIgdXBncmFkaW5nIHRvIHRoZSBsYXRlc3QgdmVyc2lvbiIpLG51bGw7ZS5fYXVkaW9Db250ZXh0PW5ldyB0fXJldHVybiBlLl9hdWRpb0NvbnRleHR9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksZS5wcm90b3R5cGUucGxheUJlbGxTb3VuZD1mdW5jdGlvbigpe3ZhciB0PWUuYXVkaW9Db250ZXh0O2lmKHQpe3ZhciByPXQuY3JlYXRlQnVmZmVyU291cmNlKCk7dC5kZWNvZGVBdWRpb0RhdGEodGhpcy5fYmFzZTY0VG9BcnJheUJ1ZmZlcih0aGlzLl9yZW1vdmVNaW1lVHlwZSh0aGlzLl9vcHRpb25zU2VydmljZS5vcHRpb25zLmJlbGxTb3VuZCkpLChmdW5jdGlvbihlKXtyLmJ1ZmZlcj1lLHIuY29ubmVjdCh0LmRlc3RpbmF0aW9uKSxyLnN0YXJ0KDApfSkpfX0sZS5wcm90b3R5cGUuX2Jhc2U2NFRvQXJyYXlCdWZmZXI9ZnVuY3Rpb24oZSl7Zm9yKHZhciB0PXdpbmRvdy5hdG9iKGUpLHI9dC5sZW5ndGgsaT1uZXcgVWludDhBcnJheShyKSxuPTA7bjxyO24rKylpW25dPXQuY2hhckNvZGVBdChuKTtyZXR1cm4gaS5idWZmZXJ9LGUucHJvdG90eXBlLl9yZW1vdmVNaW1lVHlwZT1mdW5jdGlvbihlKXtyZXR1cm4gZS5zcGxpdCgiLCIpWzFdfSxlPWkoW24oMCxvLklPcHRpb25zU2VydmljZSldLGUpfSgpO3QuU291bmRTZXJ2aWNlPXN9LDYzNDk6KGUsdCxyKT0+e09iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LCJfX2VzTW9kdWxlIix7dmFsdWU6ITB9KSx0LkNpcmN1bGFyTGlzdD12b2lkIDA7dmFyIGk9cig4NDYwKSxuPWZ1bmN0aW9uKCl7ZnVuY3Rpb24gZShlKXt0aGlzLl9tYXhMZW5ndGg9ZSx0aGlzLm9uRGVsZXRlRW1pdHRlcj1uZXcgaS5FdmVudEVtaXR0ZXIsdGhpcy5vbkluc2VydEVtaXR0ZXI9bmV3IGkuRXZlbnRFbWl0dGVyLHRoaXMub25UcmltRW1pdHRlcj1uZXcgaS5FdmVudEVtaXR0ZXIsdGhpcy5fYXJyYXk9bmV3IEFycmF5KHRoaXMuX21heExlbmd0aCksdGhpcy5fc3RhcnRJbmRleD0wLHRoaXMuX2xlbmd0aD0wfXJldHVybiBPYmplY3QuZGVmaW5lUHJvcGVydHkoZS5wcm90b3R5cGUsIm9uRGVsZXRlIix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMub25EZWxldGVFbWl0dGVyLmV2ZW50fSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLE9iamVjdC5kZWZpbmVQcm9wZXJ0eShlLnByb3RvdHlwZSwib25JbnNlcnQiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5vbkluc2VydEVtaXR0ZXIuZXZlbnR9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KGUucHJvdG90eXBlLCJvblRyaW0iLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5vblRyaW1FbWl0dGVyLmV2ZW50fSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLE9iamVjdC5kZWZpbmVQcm9wZXJ0eShlLnByb3RvdHlwZSwibWF4TGVuZ3RoIix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX21heExlbmd0aH0sc2V0OmZ1bmN0aW9uKGUpe2lmKHRoaXMuX21heExlbmd0aCE9PWUpe2Zvcih2YXIgdD1uZXcgQXJyYXkoZSkscj0wO3I8TWF0aC5taW4oZSx0aGlzLmxlbmd0aCk7cisrKXRbcl09dGhpcy5fYXJyYXlbdGhpcy5fZ2V0Q3ljbGljSW5kZXgocildO3RoaXMuX2FycmF5PXQsdGhpcy5fbWF4TGVuZ3RoPWUsdGhpcy5fc3RhcnRJbmRleD0wfX0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxPYmplY3QuZGVmaW5lUHJvcGVydHkoZS5wcm90b3R5cGUsImxlbmd0aCIse2dldDpmdW5jdGlvbigpe3JldHVybiB0aGlzLl9sZW5ndGh9LHNldDpmdW5jdGlvbihlKXtpZihlPnRoaXMuX2xlbmd0aClmb3IodmFyIHQ9dGhpcy5fbGVuZ3RoO3Q8ZTt0KyspdGhpcy5fYXJyYXlbdF09dm9pZCAwO3RoaXMuX2xlbmd0aD1lfSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLGUucHJvdG90eXBlLmdldD1mdW5jdGlvbihlKXtyZXR1cm4gdGhpcy5fYXJyYXlbdGhpcy5fZ2V0Q3ljbGljSW5kZXgoZSldfSxlLnByb3RvdHlwZS5zZXQ9ZnVuY3Rpb24oZSx0KXt0aGlzLl9hcnJheVt0aGlzLl9nZXRDeWNsaWNJbmRleChlKV09dH0sZS5wcm90b3R5cGUucHVzaD1mdW5jdGlvbihlKXt0aGlzLl9hcnJheVt0aGlzLl9nZXRDeWNsaWNJbmRleCh0aGlzLl9sZW5ndGgpXT1lLHRoaXMuX2xlbmd0aD09PXRoaXMuX21heExlbmd0aD8odGhpcy5fc3RhcnRJbmRleD0rK3RoaXMuX3N0YXJ0SW5kZXgldGhpcy5fbWF4TGVuZ3RoLHRoaXMub25UcmltRW1pdHRlci5maXJlKDEpKTp0aGlzLl9sZW5ndGgrK30sZS5wcm90b3R5cGUucmVjeWNsZT1mdW5jdGlvbigpe2lmKHRoaXMuX2xlbmd0aCE9PXRoaXMuX21heExlbmd0aCl0aHJvdyBuZXcgRXJyb3IoIkNhbiBvbmx5IHJlY3ljbGUgd2hlbiB0aGUgYnVmZmVyIGlzIGZ1bGwiKTtyZXR1cm4gdGhpcy5fc3RhcnRJbmRleD0rK3RoaXMuX3N0YXJ0SW5kZXgldGhpcy5fbWF4TGVuZ3RoLHRoaXMub25UcmltRW1pdHRlci5maXJlKDEpLHRoaXMuX2FycmF5W3RoaXMuX2dldEN5Y2xpY0luZGV4KHRoaXMuX2xlbmd0aC0xKV19LE9iamVjdC5kZWZpbmVQcm9wZXJ0eShlLnByb3RvdHlwZSwiaXNGdWxsIix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX2xlbmd0aD09PXRoaXMuX21heExlbmd0aH0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxlLnByb3RvdHlwZS5wb3A9ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fYXJyYXlbdGhpcy5fZ2V0Q3ljbGljSW5kZXgodGhpcy5fbGVuZ3RoLS0tMSldfSxlLnByb3RvdHlwZS5zcGxpY2U9ZnVuY3Rpb24oZSx0KXtmb3IodmFyIHI9W10saT0yO2k8YXJndW1lbnRzLmxlbmd0aDtpKyspcltpLTJdPWFyZ3VtZW50c1tpXTtpZih0KXtmb3IodmFyIG49ZTtuPHRoaXMuX2xlbmd0aC10O24rKyl0aGlzLl9hcnJheVt0aGlzLl9nZXRDeWNsaWNJbmRleChuKV09dGhpcy5fYXJyYXlbdGhpcy5fZ2V0Q3ljbGljSW5kZXgobit0KV07dGhpcy5fbGVuZ3RoLT10LHRoaXMub25EZWxldGVFbWl0dGVyLmZpcmUoe2luZGV4OmUsYW1vdW50OnR9KX1mb3Iobj10aGlzLl9sZW5ndGgtMTtuPj1lO24tLSl0aGlzLl9hcnJheVt0aGlzLl9nZXRDeWNsaWNJbmRleChuK3IubGVuZ3RoKV09dGhpcy5fYXJyYXlbdGhpcy5fZ2V0Q3ljbGljSW5kZXgobildO2ZvcihuPTA7bjxyLmxlbmd0aDtuKyspdGhpcy5fYXJyYXlbdGhpcy5fZ2V0Q3ljbGljSW5kZXgoZStuKV09cltuXTtpZihyLmxlbmd0aCYmdGhpcy5vbkluc2VydEVtaXR0ZXIuZmlyZSh7aW5kZXg6ZSxhbW91bnQ6ci5sZW5ndGh9KSx0aGlzLl9sZW5ndGgrci5sZW5ndGg+dGhpcy5fbWF4TGVuZ3RoKXt2YXIgbz10aGlzLl9sZW5ndGgrci5sZW5ndGgtdGhpcy5fbWF4TGVuZ3RoO3RoaXMuX3N0YXJ0SW5kZXgrPW8sdGhpcy5fbGVuZ3RoPXRoaXMuX21heExlbmd0aCx0aGlzLm9uVHJpbUVtaXR0ZXIuZmlyZShvKX1lbHNlIHRoaXMuX2xlbmd0aCs9ci5sZW5ndGh9LGUucHJvdG90eXBlLnRyaW1TdGFydD1mdW5jdGlvbihlKXtlPnRoaXMuX2xlbmd0aCYmKGU9dGhpcy5fbGVuZ3RoKSx0aGlzLl9zdGFydEluZGV4Kz1lLHRoaXMuX2xlbmd0aC09ZSx0aGlzLm9uVHJpbUVtaXR0ZXIuZmlyZShlKX0sZS5wcm90b3R5cGUuc2hpZnRFbGVtZW50cz1mdW5jdGlvbihlLHQscil7aWYoISh0PD0wKSl7aWYoZTwwfHxlPj10aGlzLl9sZW5ndGgpdGhyb3cgbmV3IEVycm9yKCJzdGFydCBhcmd1bWVudCBvdXQgb2YgcmFuZ2UiKTtpZihlK3I8MCl0aHJvdyBuZXcgRXJyb3IoIkNhbm5vdCBzaGlmdCBlbGVtZW50cyBpbiBsaXN0IGJleW9uZCBpbmRleCAwIik7aWYocj4wKXtmb3IodmFyIGk9dC0xO2k+PTA7aS0tKXRoaXMuc2V0KGUraStyLHRoaXMuZ2V0KGUraSkpO3ZhciBuPWUrdCtyLXRoaXMuX2xlbmd0aDtpZihuPjApZm9yKHRoaXMuX2xlbmd0aCs9bjt0aGlzLl9sZW5ndGg+dGhpcy5fbWF4TGVuZ3RoOyl0aGlzLl9sZW5ndGgtLSx0aGlzLl9zdGFydEluZGV4KyssdGhpcy5vblRyaW1FbWl0dGVyLmZpcmUoMSl9ZWxzZSBmb3IoaT0wO2k8dDtpKyspdGhpcy5zZXQoZStpK3IsdGhpcy5nZXQoZStpKSl9fSxlLnByb3RvdHlwZS5fZ2V0Q3ljbGljSW5kZXg9ZnVuY3Rpb24oZSl7cmV0dXJuKHRoaXMuX3N0YXJ0SW5kZXgrZSkldGhpcy5fbWF4TGVuZ3RofSxlfSgpO3QuQ2lyY3VsYXJMaXN0PW59LDE0Mzk6KGUsdCk9PntPYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC5jbG9uZT12b2lkIDAsdC5jbG9uZT1mdW5jdGlvbiBlKHQscil7aWYodm9pZCAwPT09ciYmKHI9NSksIm9iamVjdCIhPXR5cGVvZiB0KXJldHVybiB0O3ZhciBpPUFycmF5LmlzQXJyYXkodCk/W106e307Zm9yKHZhciBuIGluIHQpaVtuXT1yPD0xP3Rbbl06dFtuXSYmZSh0W25dLHItMSk7cmV0dXJuIGl9fSw4OTY5OmZ1bmN0aW9uKGUsdCxyKXt2YXIgaSxuPXRoaXMmJnRoaXMuX19leHRlbmRzfHwoaT1mdW5jdGlvbihlLHQpe3JldHVybiBpPU9iamVjdC5zZXRQcm90b3R5cGVPZnx8e19fcHJvdG9fXzpbXX1pbnN0YW5jZW9mIEFycmF5JiZmdW5jdGlvbihlLHQpe2UuX19wcm90b19fPXR9fHxmdW5jdGlvbihlLHQpe2Zvcih2YXIgciBpbiB0KU9iamVjdC5wcm90b3R5cGUuaGFzT3duUHJvcGVydHkuY2FsbCh0LHIpJiYoZVtyXT10W3JdKX0saShlLHQpfSxmdW5jdGlvbihlLHQpe2lmKCJmdW5jdGlvbiIhPXR5cGVvZiB0JiZudWxsIT09dCl0aHJvdyBuZXcgVHlwZUVycm9yKCJDbGFzcyBleHRlbmRzIHZhbHVlICIrU3RyaW5nKHQpKyIgaXMgbm90IGEgY29uc3RydWN0b3Igb3IgbnVsbCIpO2Z1bmN0aW9uIHIoKXt0aGlzLmNvbnN0cnVjdG9yPWV9aShlLHQpLGUucHJvdG90eXBlPW51bGw9PT10P09iamVjdC5jcmVhdGUodCk6KHIucHJvdG90eXBlPXQucHJvdG90eXBlLG5ldyByKX0pO09iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LCJfX2VzTW9kdWxlIix7dmFsdWU6ITB9KSx0LkNvcmVUZXJtaW5hbD12b2lkIDA7dmFyIG89cig4NDQpLHM9cigyNTg1KSxhPXIoNDM0OCksYz1yKDc4NjYpLGw9cig3NDQpLHU9cig3MzAyKSxoPXIoNjk3NSksZj1yKDg0NjApLF89cigxNzUzKSxkPXIoMzczMCkscD1yKDE0ODApLHY9cig3OTk0KSxnPXIoOTI4MikseT1yKDU0MzUpLG09cig1OTgxKSxiPSExLFM9ZnVuY3Rpb24oZSl7ZnVuY3Rpb24gdCh0KXt2YXIgcj1lLmNhbGwodGhpcyl8fHRoaXM7cmV0dXJuIHIuX29uQmluYXJ5PW5ldyBmLkV2ZW50RW1pdHRlcixyLl9vbkRhdGE9bmV3IGYuRXZlbnRFbWl0dGVyLHIuX29uTGluZUZlZWQ9bmV3IGYuRXZlbnRFbWl0dGVyLHIuX29uUmVzaXplPW5ldyBmLkV2ZW50RW1pdHRlcixyLl9vblNjcm9sbD1uZXcgZi5FdmVudEVtaXR0ZXIsci5faW5zdGFudGlhdGlvblNlcnZpY2U9bmV3IGEuSW5zdGFudGlhdGlvblNlcnZpY2Usci5vcHRpb25zU2VydmljZT1uZXcgdS5PcHRpb25zU2VydmljZSh0KSxyLl9pbnN0YW50aWF0aW9uU2VydmljZS5zZXRTZXJ2aWNlKHMuSU9wdGlvbnNTZXJ2aWNlLHIub3B0aW9uc1NlcnZpY2UpLHIuX2J1ZmZlclNlcnZpY2U9ci5yZWdpc3RlcihyLl9pbnN0YW50aWF0aW9uU2VydmljZS5jcmVhdGVJbnN0YW5jZShsLkJ1ZmZlclNlcnZpY2UpKSxyLl9pbnN0YW50aWF0aW9uU2VydmljZS5zZXRTZXJ2aWNlKHMuSUJ1ZmZlclNlcnZpY2Usci5fYnVmZmVyU2VydmljZSksci5fbG9nU2VydmljZT1yLl9pbnN0YW50aWF0aW9uU2VydmljZS5jcmVhdGVJbnN0YW5jZShjLkxvZ1NlcnZpY2UpLHIuX2luc3RhbnRpYXRpb25TZXJ2aWNlLnNldFNlcnZpY2Uocy5JTG9nU2VydmljZSxyLl9sb2dTZXJ2aWNlKSxyLmNvcmVTZXJ2aWNlPXIucmVnaXN0ZXIoci5faW5zdGFudGlhdGlvblNlcnZpY2UuY3JlYXRlSW5zdGFuY2UoaC5Db3JlU2VydmljZSwoZnVuY3Rpb24oKXtyZXR1cm4gci5zY3JvbGxUb0JvdHRvbSgpfSkpKSxyLl9pbnN0YW50aWF0aW9uU2VydmljZS5zZXRTZXJ2aWNlKHMuSUNvcmVTZXJ2aWNlLHIuY29yZVNlcnZpY2UpLHIuY29yZU1vdXNlU2VydmljZT1yLl9pbnN0YW50aWF0aW9uU2VydmljZS5jcmVhdGVJbnN0YW5jZShfLkNvcmVNb3VzZVNlcnZpY2UpLHIuX2luc3RhbnRpYXRpb25TZXJ2aWNlLnNldFNlcnZpY2Uocy5JQ29yZU1vdXNlU2VydmljZSxyLmNvcmVNb3VzZVNlcnZpY2UpLHIuX2RpcnR5Um93U2VydmljZT1yLl9pbnN0YW50aWF0aW9uU2VydmljZS5jcmVhdGVJbnN0YW5jZShkLkRpcnR5Um93U2VydmljZSksci5faW5zdGFudGlhdGlvblNlcnZpY2Uuc2V0U2VydmljZShzLklEaXJ0eVJvd1NlcnZpY2Usci5fZGlydHlSb3dTZXJ2aWNlKSxyLnVuaWNvZGVTZXJ2aWNlPXIuX2luc3RhbnRpYXRpb25TZXJ2aWNlLmNyZWF0ZUluc3RhbmNlKHAuVW5pY29kZVNlcnZpY2UpLHIuX2luc3RhbnRpYXRpb25TZXJ2aWNlLnNldFNlcnZpY2Uocy5JVW5pY29kZVNlcnZpY2Usci51bmljb2RlU2VydmljZSksci5fY2hhcnNldFNlcnZpY2U9ci5faW5zdGFudGlhdGlvblNlcnZpY2UuY3JlYXRlSW5zdGFuY2Uodi5DaGFyc2V0U2VydmljZSksci5faW5zdGFudGlhdGlvblNlcnZpY2Uuc2V0U2VydmljZShzLklDaGFyc2V0U2VydmljZSxyLl9jaGFyc2V0U2VydmljZSksci5faW5wdXRIYW5kbGVyPW5ldyB5LklucHV0SGFuZGxlcihyLl9idWZmZXJTZXJ2aWNlLHIuX2NoYXJzZXRTZXJ2aWNlLHIuY29yZVNlcnZpY2Usci5fZGlydHlSb3dTZXJ2aWNlLHIuX2xvZ1NlcnZpY2Usci5vcHRpb25zU2VydmljZSxyLmNvcmVNb3VzZVNlcnZpY2Usci51bmljb2RlU2VydmljZSksci5yZWdpc3RlcigoMCxmLmZvcndhcmRFdmVudCkoci5faW5wdXRIYW5kbGVyLm9uTGluZUZlZWQsci5fb25MaW5lRmVlZCkpLHIucmVnaXN0ZXIoci5faW5wdXRIYW5kbGVyKSxyLnJlZ2lzdGVyKCgwLGYuZm9yd2FyZEV2ZW50KShyLl9idWZmZXJTZXJ2aWNlLm9uUmVzaXplLHIuX29uUmVzaXplKSksci5yZWdpc3RlcigoMCxmLmZvcndhcmRFdmVudCkoci5jb3JlU2VydmljZS5vbkRhdGEsci5fb25EYXRhKSksci5yZWdpc3RlcigoMCxmLmZvcndhcmRFdmVudCkoci5jb3JlU2VydmljZS5vbkJpbmFyeSxyLl9vbkJpbmFyeSkpLHIucmVnaXN0ZXIoci5vcHRpb25zU2VydmljZS5vbk9wdGlvbkNoYW5nZSgoZnVuY3Rpb24oZSl7cmV0dXJuIHIuX3VwZGF0ZU9wdGlvbnMoZSl9KSkpLHIucmVnaXN0ZXIoci5fYnVmZmVyU2VydmljZS5vblNjcm9sbCgoZnVuY3Rpb24oZSl7ci5fb25TY3JvbGwuZmlyZSh7cG9zaXRpb246ci5fYnVmZmVyU2VydmljZS5idWZmZXIueWRpc3Asc291cmNlOjB9KSxyLl9kaXJ0eVJvd1NlcnZpY2UubWFya1JhbmdlRGlydHkoci5fYnVmZmVyU2VydmljZS5idWZmZXIuc2Nyb2xsVG9wLHIuX2J1ZmZlclNlcnZpY2UuYnVmZmVyLnNjcm9sbEJvdHRvbSl9KSkpLHIucmVnaXN0ZXIoci5faW5wdXRIYW5kbGVyLm9uU2Nyb2xsKChmdW5jdGlvbihlKXtyLl9vblNjcm9sbC5maXJlKHtwb3NpdGlvbjpyLl9idWZmZXJTZXJ2aWNlLmJ1ZmZlci55ZGlzcCxzb3VyY2U6MH0pLHIuX2RpcnR5Um93U2VydmljZS5tYXJrUmFuZ2VEaXJ0eShyLl9idWZmZXJTZXJ2aWNlLmJ1ZmZlci5zY3JvbGxUb3Asci5fYnVmZmVyU2VydmljZS5idWZmZXIuc2Nyb2xsQm90dG9tKX0pKSksci5fd3JpdGVCdWZmZXI9bmV3IG0uV3JpdGVCdWZmZXIoKGZ1bmN0aW9uKGUsdCl7cmV0dXJuIHIuX2lucHV0SGFuZGxlci5wYXJzZShlLHQpfSkpLHJ9cmV0dXJuIG4odCxlKSxPYmplY3QuZGVmaW5lUHJvcGVydHkodC5wcm90b3R5cGUsIm9uQmluYXJ5Iix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX29uQmluYXJ5LmV2ZW50fSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLE9iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LnByb3RvdHlwZSwib25EYXRhIix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX29uRGF0YS5ldmVudH0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxPYmplY3QuZGVmaW5lUHJvcGVydHkodC5wcm90b3R5cGUsIm9uTGluZUZlZWQiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fb25MaW5lRmVlZC5ldmVudH0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxPYmplY3QuZGVmaW5lUHJvcGVydHkodC5wcm90b3R5cGUsIm9uUmVzaXplIix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX29uUmVzaXplLmV2ZW50fSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLE9iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LnByb3RvdHlwZSwib25TY3JvbGwiLHtnZXQ6ZnVuY3Rpb24oKXt2YXIgZT10aGlzO3JldHVybiB0aGlzLl9vblNjcm9sbEFwaXx8KHRoaXMuX29uU2Nyb2xsQXBpPW5ldyBmLkV2ZW50RW1pdHRlcix0aGlzLnJlZ2lzdGVyKHRoaXMuX29uU2Nyb2xsLmV2ZW50KChmdW5jdGlvbih0KXt2YXIgcjtudWxsPT09KHI9ZS5fb25TY3JvbGxBcGkpfHx2b2lkIDA9PT1yfHxyLmZpcmUodC5wb3NpdGlvbil9KSkpKSx0aGlzLl9vblNjcm9sbEFwaS5ldmVudH0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxPYmplY3QuZGVmaW5lUHJvcGVydHkodC5wcm90b3R5cGUsImNvbHMiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fYnVmZmVyU2VydmljZS5jb2xzfSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLE9iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LnByb3RvdHlwZSwicm93cyIse2dldDpmdW5jdGlvbigpe3JldHVybiB0aGlzLl9idWZmZXJTZXJ2aWNlLnJvd3N9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KHQucHJvdG90eXBlLCJidWZmZXJzIix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX2J1ZmZlclNlcnZpY2UuYnVmZmVyc30sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxPYmplY3QuZGVmaW5lUHJvcGVydHkodC5wcm90b3R5cGUsIm9wdGlvbnMiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5vcHRpb25zU2VydmljZS5vcHRpb25zfSxzZXQ6ZnVuY3Rpb24oZSl7Zm9yKHZhciB0IGluIGUpdGhpcy5vcHRpb25zU2VydmljZS5vcHRpb25zW3RdPWVbdF19LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksdC5wcm90b3R5cGUuZGlzcG9zZT1mdW5jdGlvbigpe3ZhciB0O3RoaXMuX2lzRGlzcG9zZWR8fChlLnByb3RvdHlwZS5kaXNwb3NlLmNhbGwodGhpcyksbnVsbD09PSh0PXRoaXMuX3dpbmRvd3NNb2RlKXx8dm9pZCAwPT09dHx8dC5kaXNwb3NlKCksdGhpcy5fd2luZG93c01vZGU9dm9pZCAwKX0sdC5wcm90b3R5cGUud3JpdGU9ZnVuY3Rpb24oZSx0KXt0aGlzLl93cml0ZUJ1ZmZlci53cml0ZShlLHQpfSx0LnByb3RvdHlwZS53cml0ZVN5bmM9ZnVuY3Rpb24oZSx0KXt0aGlzLl9sb2dTZXJ2aWNlLmxvZ0xldmVsPD1zLkxvZ0xldmVsRW51bS5XQVJOJiYhYiYmKHRoaXMuX2xvZ1NlcnZpY2Uud2Fybigid3JpdGVTeW5jIGlzIHVucmVsaWFibGUgYW5kIHdpbGwgYmUgcmVtb3ZlZCBzb29uLiIpLGI9ITApLHRoaXMuX3dyaXRlQnVmZmVyLndyaXRlU3luYyhlLHQpfSx0LnByb3RvdHlwZS5yZXNpemU9ZnVuY3Rpb24oZSx0KXtpc05hTihlKXx8aXNOYU4odCl8fChlPU1hdGgubWF4KGUsbC5NSU5JTVVNX0NPTFMpLHQ9TWF0aC5tYXgodCxsLk1JTklNVU1fUk9XUyksdGhpcy5fYnVmZmVyU2VydmljZS5yZXNpemUoZSx0KSl9LHQucHJvdG90eXBlLnNjcm9sbD1mdW5jdGlvbihlLHQpe3ZvaWQgMD09PXQmJih0PSExKSx0aGlzLl9idWZmZXJTZXJ2aWNlLnNjcm9sbChlLHQpfSx0LnByb3RvdHlwZS5zY3JvbGxMaW5lcz1mdW5jdGlvbihlLHQscil7dGhpcy5fYnVmZmVyU2VydmljZS5zY3JvbGxMaW5lcyhlLHQscil9LHQucHJvdG90eXBlLnNjcm9sbFBhZ2VzPWZ1bmN0aW9uKGUpe3RoaXMuX2J1ZmZlclNlcnZpY2Uuc2Nyb2xsUGFnZXMoZSl9LHQucHJvdG90eXBlLnNjcm9sbFRvVG9wPWZ1bmN0aW9uKCl7dGhpcy5fYnVmZmVyU2VydmljZS5zY3JvbGxUb1RvcCgpfSx0LnByb3RvdHlwZS5zY3JvbGxUb0JvdHRvbT1mdW5jdGlvbigpe3RoaXMuX2J1ZmZlclNlcnZpY2Uuc2Nyb2xsVG9Cb3R0b20oKX0sdC5wcm90b3R5cGUuc2Nyb2xsVG9MaW5lPWZ1bmN0aW9uKGUpe3RoaXMuX2J1ZmZlclNlcnZpY2Uuc2Nyb2xsVG9MaW5lKGUpfSx0LnByb3RvdHlwZS5yZWdpc3RlckVzY0hhbmRsZXI9ZnVuY3Rpb24oZSx0KXtyZXR1cm4gdGhpcy5faW5wdXRIYW5kbGVyLnJlZ2lzdGVyRXNjSGFuZGxlcihlLHQpfSx0LnByb3RvdHlwZS5yZWdpc3RlckRjc0hhbmRsZXI9ZnVuY3Rpb24oZSx0KXtyZXR1cm4gdGhpcy5faW5wdXRIYW5kbGVyLnJlZ2lzdGVyRGNzSGFuZGxlcihlLHQpfSx0LnByb3RvdHlwZS5yZWdpc3RlckNzaUhhbmRsZXI9ZnVuY3Rpb24oZSx0KXtyZXR1cm4gdGhpcy5faW5wdXRIYW5kbGVyLnJlZ2lzdGVyQ3NpSGFuZGxlcihlLHQpfSx0LnByb3RvdHlwZS5yZWdpc3Rlck9zY0hhbmRsZXI9ZnVuY3Rpb24oZSx0KXtyZXR1cm4gdGhpcy5faW5wdXRIYW5kbGVyLnJlZ2lzdGVyT3NjSGFuZGxlcihlLHQpfSx0LnByb3RvdHlwZS5fc2V0dXA9ZnVuY3Rpb24oKXt0aGlzLm9wdGlvbnNTZXJ2aWNlLm9wdGlvbnMud2luZG93c01vZGUmJnRoaXMuX2VuYWJsZVdpbmRvd3NNb2RlKCl9LHQucHJvdG90eXBlLnJlc2V0PWZ1bmN0aW9uKCl7dGhpcy5faW5wdXRIYW5kbGVyLnJlc2V0KCksdGhpcy5fYnVmZmVyU2VydmljZS5yZXNldCgpLHRoaXMuX2NoYXJzZXRTZXJ2aWNlLnJlc2V0KCksdGhpcy5jb3JlU2VydmljZS5yZXNldCgpLHRoaXMuY29yZU1vdXNlU2VydmljZS5yZXNldCgpfSx0LnByb3RvdHlwZS5fdXBkYXRlT3B0aW9ucz1mdW5jdGlvbihlKXt2YXIgdDtzd2l0Y2goZSl7Y2FzZSJzY3JvbGxiYWNrIjp0aGlzLmJ1ZmZlcnMucmVzaXplKHRoaXMuY29scyx0aGlzLnJvd3MpO2JyZWFrO2Nhc2Uid2luZG93c01vZGUiOnRoaXMub3B0aW9uc1NlcnZpY2Uub3B0aW9ucy53aW5kb3dzTW9kZT90aGlzLl9lbmFibGVXaW5kb3dzTW9kZSgpOihudWxsPT09KHQ9dGhpcy5fd2luZG93c01vZGUpfHx2b2lkIDA9PT10fHx0LmRpc3Bvc2UoKSx0aGlzLl93aW5kb3dzTW9kZT12b2lkIDApfX0sdC5wcm90b3R5cGUuX2VuYWJsZVdpbmRvd3NNb2RlPWZ1bmN0aW9uKCl7dmFyIGU9dGhpcztpZighdGhpcy5fd2luZG93c01vZGUpe3ZhciB0PVtdO3QucHVzaCh0aGlzLm9uTGluZUZlZWQoZy51cGRhdGVXaW5kb3dzTW9kZVdyYXBwZWRTdGF0ZS5iaW5kKG51bGwsdGhpcy5fYnVmZmVyU2VydmljZSkpKSx0LnB1c2godGhpcy5yZWdpc3RlckNzaUhhbmRsZXIoe2ZpbmFsOiJIIn0sKGZ1bmN0aW9uKCl7cmV0dXJuKDAsZy51cGRhdGVXaW5kb3dzTW9kZVdyYXBwZWRTdGF0ZSkoZS5fYnVmZmVyU2VydmljZSksITF9KSkpLHRoaXMuX3dpbmRvd3NNb2RlPXtkaXNwb3NlOmZ1bmN0aW9uKCl7Zm9yKHZhciBlPTAscj10O2U8ci5sZW5ndGg7ZSsrKXJbZV0uZGlzcG9zZSgpfX19fSx0fShvLkRpc3Bvc2FibGUpO3QuQ29yZVRlcm1pbmFsPVN9LDg0NjA6KGUsdCk9PntPYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC5mb3J3YXJkRXZlbnQ9dC5FdmVudEVtaXR0ZXI9dm9pZCAwO3ZhciByPWZ1bmN0aW9uKCl7ZnVuY3Rpb24gZSgpe3RoaXMuX2xpc3RlbmVycz1bXSx0aGlzLl9kaXNwb3NlZD0hMX1yZXR1cm4gT2JqZWN0LmRlZmluZVByb3BlcnR5KGUucHJvdG90eXBlLCJldmVudCIse2dldDpmdW5jdGlvbigpe3ZhciBlPXRoaXM7cmV0dXJuIHRoaXMuX2V2ZW50fHwodGhpcy5fZXZlbnQ9ZnVuY3Rpb24odCl7cmV0dXJuIGUuX2xpc3RlbmVycy5wdXNoKHQpLHtkaXNwb3NlOmZ1bmN0aW9uKCl7aWYoIWUuX2Rpc3Bvc2VkKWZvcih2YXIgcj0wO3I8ZS5fbGlzdGVuZXJzLmxlbmd0aDtyKyspaWYoZS5fbGlzdGVuZXJzW3JdPT09dClyZXR1cm4gdm9pZCBlLl9saXN0ZW5lcnMuc3BsaWNlKHIsMSl9fX0pLHRoaXMuX2V2ZW50fSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLGUucHJvdG90eXBlLmZpcmU9ZnVuY3Rpb24oZSx0KXtmb3IodmFyIHI9W10saT0wO2k8dGhpcy5fbGlzdGVuZXJzLmxlbmd0aDtpKyspci5wdXNoKHRoaXMuX2xpc3RlbmVyc1tpXSk7Zm9yKGk9MDtpPHIubGVuZ3RoO2krKylyW2ldLmNhbGwodm9pZCAwLGUsdCl9LGUucHJvdG90eXBlLmRpc3Bvc2U9ZnVuY3Rpb24oKXt0aGlzLl9saXN0ZW5lcnMmJih0aGlzLl9saXN0ZW5lcnMubGVuZ3RoPTApLHRoaXMuX2Rpc3Bvc2VkPSEwfSxlfSgpO3QuRXZlbnRFbWl0dGVyPXIsdC5mb3J3YXJkRXZlbnQ9ZnVuY3Rpb24oZSx0KXtyZXR1cm4gZSgoZnVuY3Rpb24oZSl7cmV0dXJuIHQuZmlyZShlKX0pKX19LDU0MzU6ZnVuY3Rpb24oZSx0LHIpe3ZhciBpLG49dGhpcyYmdGhpcy5fX2V4dGVuZHN8fChpPWZ1bmN0aW9uKGUsdCl7cmV0dXJuIGk9T2JqZWN0LnNldFByb3RvdHlwZU9mfHx7X19wcm90b19fOltdfWluc3RhbmNlb2YgQXJyYXkmJmZ1bmN0aW9uKGUsdCl7ZS5fX3Byb3RvX189dH18fGZ1bmN0aW9uKGUsdCl7Zm9yKHZhciByIGluIHQpT2JqZWN0LnByb3RvdHlwZS5oYXNPd25Qcm9wZXJ0eS5jYWxsKHQscikmJihlW3JdPXRbcl0pfSxpKGUsdCl9LGZ1bmN0aW9uKGUsdCl7aWYoImZ1bmN0aW9uIiE9dHlwZW9mIHQmJm51bGwhPT10KXRocm93IG5ldyBUeXBlRXJyb3IoIkNsYXNzIGV4dGVuZHMgdmFsdWUgIitTdHJpbmcodCkrIiBpcyBub3QgYSBjb25zdHJ1Y3RvciBvciBudWxsIik7ZnVuY3Rpb24gcigpe3RoaXMuY29uc3RydWN0b3I9ZX1pKGUsdCksZS5wcm90b3R5cGU9bnVsbD09PXQ/T2JqZWN0LmNyZWF0ZSh0KTooci5wcm90b3R5cGU9dC5wcm90b3R5cGUsbmV3IHIpfSk7T2JqZWN0LmRlZmluZVByb3BlcnR5KHQsIl9fZXNNb2R1bGUiLHt2YWx1ZTohMH0pLHQuSW5wdXRIYW5kbGVyPXQuV2luZG93c09wdGlvbnNSZXBvcnRUeXBlPXZvaWQgMDt2YXIgbyxzPXIoMjU4NCksYT1yKDcxMTYpLGM9cigyMDE1KSxsPXIoODQ0KSx1PXIoODI3MyksaD1yKDQ4MiksZj1yKDg0MzcpLF89cig4NDYwKSxkPXIoNjQzKSxwPXIoNTExKSx2PXIoMzczNCksZz1yKDI1ODUpLHk9cig2MjQyKSxtPXIoNjM1MSksYj1yKDU5NDEpLFM9eyIoIjowLCIpIjoxLCIqIjoyLCIrIjozLCItIjoxLCIuIjoyfSxDPTEzMTA3MjtmdW5jdGlvbiB3KGUsdCl7aWYoZT4yNClyZXR1cm4gdC5zZXRXaW5MaW5lc3x8ITE7c3dpdGNoKGUpe2Nhc2UgMTpyZXR1cm4hIXQucmVzdG9yZVdpbjtjYXNlIDI6cmV0dXJuISF0Lm1pbmltaXplV2luO2Nhc2UgMzpyZXR1cm4hIXQuc2V0V2luUG9zaXRpb247Y2FzZSA0OnJldHVybiEhdC5zZXRXaW5TaXplUGl4ZWxzO2Nhc2UgNTpyZXR1cm4hIXQucmFpc2VXaW47Y2FzZSA2OnJldHVybiEhdC5sb3dlcldpbjtjYXNlIDc6cmV0dXJuISF0LnJlZnJlc2hXaW47Y2FzZSA4OnJldHVybiEhdC5zZXRXaW5TaXplQ2hhcnM7Y2FzZSA5OnJldHVybiEhdC5tYXhpbWl6ZVdpbjtjYXNlIDEwOnJldHVybiEhdC5mdWxsc2NyZWVuV2luO2Nhc2UgMTE6cmV0dXJuISF0LmdldFdpblN0YXRlO2Nhc2UgMTM6cmV0dXJuISF0LmdldFdpblBvc2l0aW9uO2Nhc2UgMTQ6cmV0dXJuISF0LmdldFdpblNpemVQaXhlbHM7Y2FzZSAxNTpyZXR1cm4hIXQuZ2V0U2NyZWVuU2l6ZVBpeGVscztjYXNlIDE2OnJldHVybiEhdC5nZXRDZWxsU2l6ZVBpeGVscztjYXNlIDE4OnJldHVybiEhdC5nZXRXaW5TaXplQ2hhcnM7Y2FzZSAxOTpyZXR1cm4hIXQuZ2V0U2NyZWVuU2l6ZUNoYXJzO2Nhc2UgMjA6cmV0dXJuISF0LmdldEljb25UaXRsZTtjYXNlIDIxOnJldHVybiEhdC5nZXRXaW5UaXRsZTtjYXNlIDIyOnJldHVybiEhdC5wdXNoVGl0bGU7Y2FzZSAyMzpyZXR1cm4hIXQucG9wVGl0bGU7Y2FzZSAyNDpyZXR1cm4hIXQuc2V0V2luTGluZXN9cmV0dXJuITF9IWZ1bmN0aW9uKGUpe2VbZS5HRVRfV0lOX1NJWkVfUElYRUxTPTBdPSJHRVRfV0lOX1NJWkVfUElYRUxTIixlW2UuR0VUX0NFTExfU0laRV9QSVhFTFM9MV09IkdFVF9DRUxMX1NJWkVfUElYRUxTIn0obz10LldpbmRvd3NPcHRpb25zUmVwb3J0VHlwZXx8KHQuV2luZG93c09wdGlvbnNSZXBvcnRUeXBlPXt9KSk7dmFyIEw9ZnVuY3Rpb24oKXtmdW5jdGlvbiBlKGUsdCxyLGkpe3RoaXMuX2J1ZmZlclNlcnZpY2U9ZSx0aGlzLl9jb3JlU2VydmljZT10LHRoaXMuX2xvZ1NlcnZpY2U9cix0aGlzLl9vcHRpb25zU2VydmljZT1pLHRoaXMuX2RhdGE9bmV3IFVpbnQzMkFycmF5KDApfXJldHVybiBlLnByb3RvdHlwZS5ob29rPWZ1bmN0aW9uKGUpe3RoaXMuX2RhdGE9bmV3IFVpbnQzMkFycmF5KDApfSxlLnByb3RvdHlwZS5wdXQ9ZnVuY3Rpb24oZSx0LHIpe3RoaXMuX2RhdGE9KDAsdS5jb25jYXQpKHRoaXMuX2RhdGEsZS5zdWJhcnJheSh0LHIpKX0sZS5wcm90b3R5cGUudW5ob29rPWZ1bmN0aW9uKGUpe2lmKCFlKXJldHVybiB0aGlzLl9kYXRhPW5ldyBVaW50MzJBcnJheSgwKSwhMDt2YXIgdD0oMCxoLnV0ZjMyVG9TdHJpbmcpKHRoaXMuX2RhdGEpO3N3aXRjaCh0aGlzLl9kYXRhPW5ldyBVaW50MzJBcnJheSgwKSx0KXtjYXNlJyJxJzp0aGlzLl9jb3JlU2VydmljZS50cmlnZ2VyRGF0YUV2ZW50KHMuQzAuRVNDKydQMSRyMCJxJytzLkMwLkVTQysiXFwiKTticmVhaztjYXNlJyJwJzp0aGlzLl9jb3JlU2VydmljZS50cmlnZ2VyRGF0YUV2ZW50KHMuQzAuRVNDKydQMSRyNjE7MSJwJytzLkMwLkVTQysiXFwiKTticmVhaztjYXNlInIiOnZhciByPXRoaXMuX2J1ZmZlclNlcnZpY2UuYnVmZmVyLnNjcm9sbFRvcCsxKyI7IisodGhpcy5fYnVmZmVyU2VydmljZS5idWZmZXIuc2Nyb2xsQm90dG9tKzEpKyJyIjt0aGlzLl9jb3JlU2VydmljZS50cmlnZ2VyRGF0YUV2ZW50KHMuQzAuRVNDKyJQMSRyIityK3MuQzAuRVNDKyJcXCIpO2JyZWFrO2Nhc2UibSI6dGhpcy5fY29yZVNlcnZpY2UudHJpZ2dlckRhdGFFdmVudChzLkMwLkVTQysiUDEkcjBtIitzLkMwLkVTQysiXFwiKTticmVhaztjYXNlIiBxIjp2YXIgaT17YmxvY2s6Mix1bmRlcmxpbmU6NCxiYXI6Nn1bdGhpcy5fb3B0aW9uc1NlcnZpY2Uub3B0aW9ucy5jdXJzb3JTdHlsZV07aS09dGhpcy5fb3B0aW9uc1NlcnZpY2Uub3B0aW9ucy5jdXJzb3JCbGluaz8xOjAsdGhpcy5fY29yZVNlcnZpY2UudHJpZ2dlckRhdGFFdmVudChzLkMwLkVTQysiUDEkciIraSsiIHEiK3MuQzAuRVNDKyJcXCIpO2JyZWFrO2RlZmF1bHQ6dGhpcy5fbG9nU2VydmljZS5kZWJ1ZygiVW5rbm93biBEQ1MgJHEgJXMiLHQpLHRoaXMuX2NvcmVTZXJ2aWNlLnRyaWdnZXJEYXRhRXZlbnQocy5DMC5FU0MrIlAwJHIiK3MuQzAuRVNDKyJcXCIpfXJldHVybiEwfSxlfSgpLEU9ZnVuY3Rpb24oZSl7ZnVuY3Rpb24gdCh0LHIsaSxuLG8sbCx1LGQsdil7dm9pZCAwPT09diYmKHY9bmV3IGMuRXNjYXBlU2VxdWVuY2VQYXJzZXIpO3ZhciBnPWUuY2FsbCh0aGlzKXx8dGhpcztnLl9idWZmZXJTZXJ2aWNlPXQsZy5fY2hhcnNldFNlcnZpY2U9cixnLl9jb3JlU2VydmljZT1pLGcuX2RpcnR5Um93U2VydmljZT1uLGcuX2xvZ1NlcnZpY2U9byxnLl9vcHRpb25zU2VydmljZT1sLGcuX2NvcmVNb3VzZVNlcnZpY2U9dSxnLl91bmljb2RlU2VydmljZT1kLGcuX3BhcnNlcj12LGcuX3BhcnNlQnVmZmVyPW5ldyBVaW50MzJBcnJheSg0MDk2KSxnLl9zdHJpbmdEZWNvZGVyPW5ldyBoLlN0cmluZ1RvVXRmMzIsZy5fdXRmOERlY29kZXI9bmV3IGguVXRmOFRvVXRmMzIsZy5fd29ya0NlbGw9bmV3IHAuQ2VsbERhdGEsZy5fd2luZG93VGl0bGU9IiIsZy5faWNvbk5hbWU9IiIsZy5fd2luZG93VGl0bGVTdGFjaz1bXSxnLl9pY29uTmFtZVN0YWNrPVtdLGcuX2N1ckF0dHJEYXRhPWYuREVGQVVMVF9BVFRSX0RBVEEuY2xvbmUoKSxnLl9lcmFzZUF0dHJEYXRhSW50ZXJuYWw9Zi5ERUZBVUxUX0FUVFJfREFUQS5jbG9uZSgpLGcuX29uUmVxdWVzdEJlbGw9bmV3IF8uRXZlbnRFbWl0dGVyLGcuX29uUmVxdWVzdFJlZnJlc2hSb3dzPW5ldyBfLkV2ZW50RW1pdHRlcixnLl9vblJlcXVlc3RSZXNldD1uZXcgXy5FdmVudEVtaXR0ZXIsZy5fb25SZXF1ZXN0U2VuZEZvY3VzPW5ldyBfLkV2ZW50RW1pdHRlcixnLl9vblJlcXVlc3RTeW5jU2Nyb2xsQmFyPW5ldyBfLkV2ZW50RW1pdHRlcixnLl9vblJlcXVlc3RXaW5kb3dzT3B0aW9uc1JlcG9ydD1uZXcgXy5FdmVudEVtaXR0ZXIsZy5fb25BMTF5Q2hhcj1uZXcgXy5FdmVudEVtaXR0ZXIsZy5fb25BMTF5VGFiPW5ldyBfLkV2ZW50RW1pdHRlcixnLl9vbkN1cnNvck1vdmU9bmV3IF8uRXZlbnRFbWl0dGVyLGcuX29uTGluZUZlZWQ9bmV3IF8uRXZlbnRFbWl0dGVyLGcuX29uU2Nyb2xsPW5ldyBfLkV2ZW50RW1pdHRlcixnLl9vblRpdGxlQ2hhbmdlPW5ldyBfLkV2ZW50RW1pdHRlcixnLl9vbkNvbG9yPW5ldyBfLkV2ZW50RW1pdHRlcixnLl9wYXJzZVN0YWNrPXtwYXVzZWQ6ITEsY3Vyc29yU3RhcnRYOjAsY3Vyc29yU3RhcnRZOjAsZGVjb2RlZExlbmd0aDowLHBvc2l0aW9uOjB9LGcuX3NwZWNpYWxDb2xvcnM9WzI1NiwyNTcsMjU4XSxnLnJlZ2lzdGVyKGcuX3BhcnNlciksZy5fYWN0aXZlQnVmZmVyPWcuX2J1ZmZlclNlcnZpY2UuYnVmZmVyLGcucmVnaXN0ZXIoZy5fYnVmZmVyU2VydmljZS5idWZmZXJzLm9uQnVmZmVyQWN0aXZhdGUoKGZ1bmN0aW9uKGUpe3JldHVybiBnLl9hY3RpdmVCdWZmZXI9ZS5hY3RpdmVCdWZmZXJ9KSkpLGcuX3BhcnNlci5zZXRDc2lIYW5kbGVyRmFsbGJhY2soKGZ1bmN0aW9uKGUsdCl7Zy5fbG9nU2VydmljZS5kZWJ1ZygiVW5rbm93biBDU0kgY29kZTogIix7aWRlbnRpZmllcjpnLl9wYXJzZXIuaWRlbnRUb1N0cmluZyhlKSxwYXJhbXM6dC50b0FycmF5KCl9KX0pKSxnLl9wYXJzZXIuc2V0RXNjSGFuZGxlckZhbGxiYWNrKChmdW5jdGlvbihlKXtnLl9sb2dTZXJ2aWNlLmRlYnVnKCJVbmtub3duIEVTQyBjb2RlOiAiLHtpZGVudGlmaWVyOmcuX3BhcnNlci5pZGVudFRvU3RyaW5nKGUpfSl9KSksZy5fcGFyc2VyLnNldEV4ZWN1dGVIYW5kbGVyRmFsbGJhY2soKGZ1bmN0aW9uKGUpe2cuX2xvZ1NlcnZpY2UuZGVidWcoIlVua25vd24gRVhFQ1VURSBjb2RlOiAiLHtjb2RlOmV9KX0pKSxnLl9wYXJzZXIuc2V0T3NjSGFuZGxlckZhbGxiYWNrKChmdW5jdGlvbihlLHQscil7Zy5fbG9nU2VydmljZS5kZWJ1ZygiVW5rbm93biBPU0MgY29kZTogIix7aWRlbnRpZmllcjplLGFjdGlvbjp0LGRhdGE6cn0pfSkpLGcuX3BhcnNlci5zZXREY3NIYW5kbGVyRmFsbGJhY2soKGZ1bmN0aW9uKGUsdCxyKXsiSE9PSyI9PT10JiYocj1yLnRvQXJyYXkoKSksZy5fbG9nU2VydmljZS5kZWJ1ZygiVW5rbm93biBEQ1MgY29kZTogIix7aWRlbnRpZmllcjpnLl9wYXJzZXIuaWRlbnRUb1N0cmluZyhlKSxhY3Rpb246dCxwYXlsb2FkOnJ9KX0pKSxnLl9wYXJzZXIuc2V0UHJpbnRIYW5kbGVyKChmdW5jdGlvbihlLHQscil7cmV0dXJuIGcucHJpbnQoZSx0LHIpfSkpLGcuX3BhcnNlci5yZWdpc3RlckNzaUhhbmRsZXIoe2ZpbmFsOiJAIn0sKGZ1bmN0aW9uKGUpe3JldHVybiBnLmluc2VydENoYXJzKGUpfSkpLGcuX3BhcnNlci5yZWdpc3RlckNzaUhhbmRsZXIoe2ludGVybWVkaWF0ZXM6IiAiLGZpbmFsOiJAIn0sKGZ1bmN0aW9uKGUpe3JldHVybiBnLnNjcm9sbExlZnQoZSl9KSksZy5fcGFyc2VyLnJlZ2lzdGVyQ3NpSGFuZGxlcih7ZmluYWw6IkEifSwoZnVuY3Rpb24oZSl7cmV0dXJuIGcuY3Vyc29yVXAoZSl9KSksZy5fcGFyc2VyLnJlZ2lzdGVyQ3NpSGFuZGxlcih7aW50ZXJtZWRpYXRlczoiICIsZmluYWw6IkEifSwoZnVuY3Rpb24oZSl7cmV0dXJuIGcuc2Nyb2xsUmlnaHQoZSl9KSksZy5fcGFyc2VyLnJlZ2lzdGVyQ3NpSGFuZGxlcih7ZmluYWw6IkIifSwoZnVuY3Rpb24oZSl7cmV0dXJuIGcuY3Vyc29yRG93bihlKX0pKSxnLl9wYXJzZXIucmVnaXN0ZXJDc2lIYW5kbGVyKHtmaW5hbDoiQyJ9LChmdW5jdGlvbihlKXtyZXR1cm4gZy5jdXJzb3JGb3J3YXJkKGUpfSkpLGcuX3BhcnNlci5yZWdpc3RlckNzaUhhbmRsZXIoe2ZpbmFsOiJEIn0sKGZ1bmN0aW9uKGUpe3JldHVybiBnLmN1cnNvckJhY2t3YXJkKGUpfSkpLGcuX3BhcnNlci5yZWdpc3RlckNzaUhhbmRsZXIoe2ZpbmFsOiJFIn0sKGZ1bmN0aW9uKGUpe3JldHVybiBnLmN1cnNvck5leHRMaW5lKGUpfSkpLGcuX3BhcnNlci5yZWdpc3RlckNzaUhhbmRsZXIoe2ZpbmFsOiJGIn0sKGZ1bmN0aW9uKGUpe3JldHVybiBnLmN1cnNvclByZWNlZGluZ0xpbmUoZSl9KSksZy5fcGFyc2VyLnJlZ2lzdGVyQ3NpSGFuZGxlcih7ZmluYWw6IkcifSwoZnVuY3Rpb24oZSl7cmV0dXJuIGcuY3Vyc29yQ2hhckFic29sdXRlKGUpfSkpLGcuX3BhcnNlci5yZWdpc3RlckNzaUhhbmRsZXIoe2ZpbmFsOiJIIn0sKGZ1bmN0aW9uKGUpe3JldHVybiBnLmN1cnNvclBvc2l0aW9uKGUpfSkpLGcuX3BhcnNlci5yZWdpc3RlckNzaUhhbmRsZXIoe2ZpbmFsOiJJIn0sKGZ1bmN0aW9uKGUpe3JldHVybiBnLmN1cnNvckZvcndhcmRUYWIoZSl9KSksZy5fcGFyc2VyLnJlZ2lzdGVyQ3NpSGFuZGxlcih7ZmluYWw6IkoifSwoZnVuY3Rpb24oZSl7cmV0dXJuIGcuZXJhc2VJbkRpc3BsYXkoZSl9KSksZy5fcGFyc2VyLnJlZ2lzdGVyQ3NpSGFuZGxlcih7cHJlZml4OiI/IixmaW5hbDoiSiJ9LChmdW5jdGlvbihlKXtyZXR1cm4gZy5lcmFzZUluRGlzcGxheShlKX0pKSxnLl9wYXJzZXIucmVnaXN0ZXJDc2lIYW5kbGVyKHtmaW5hbDoiSyJ9LChmdW5jdGlvbihlKXtyZXR1cm4gZy5lcmFzZUluTGluZShlKX0pKSxnLl9wYXJzZXIucmVnaXN0ZXJDc2lIYW5kbGVyKHtwcmVmaXg6Ij8iLGZpbmFsOiJLIn0sKGZ1bmN0aW9uKGUpe3JldHVybiBnLmVyYXNlSW5MaW5lKGUpfSkpLGcuX3BhcnNlci5yZWdpc3RlckNzaUhhbmRsZXIoe2ZpbmFsOiJMIn0sKGZ1bmN0aW9uKGUpe3JldHVybiBnLmluc2VydExpbmVzKGUpfSkpLGcuX3BhcnNlci5yZWdpc3RlckNzaUhhbmRsZXIoe2ZpbmFsOiJNIn0sKGZ1bmN0aW9uKGUpe3JldHVybiBnLmRlbGV0ZUxpbmVzKGUpfSkpLGcuX3BhcnNlci5yZWdpc3RlckNzaUhhbmRsZXIoe2ZpbmFsOiJQIn0sKGZ1bmN0aW9uKGUpe3JldHVybiBnLmRlbGV0ZUNoYXJzKGUpfSkpLGcuX3BhcnNlci5yZWdpc3RlckNzaUhhbmRsZXIoe2ZpbmFsOiJTIn0sKGZ1bmN0aW9uKGUpe3JldHVybiBnLnNjcm9sbFVwKGUpfSkpLGcuX3BhcnNlci5yZWdpc3RlckNzaUhhbmRsZXIoe2ZpbmFsOiJUIn0sKGZ1bmN0aW9uKGUpe3JldHVybiBnLnNjcm9sbERvd24oZSl9KSksZy5fcGFyc2VyLnJlZ2lzdGVyQ3NpSGFuZGxlcih7ZmluYWw6IlgifSwoZnVuY3Rpb24oZSl7cmV0dXJuIGcuZXJhc2VDaGFycyhlKX0pKSxnLl9wYXJzZXIucmVnaXN0ZXJDc2lIYW5kbGVyKHtmaW5hbDoiWiJ9LChmdW5jdGlvbihlKXtyZXR1cm4gZy5jdXJzb3JCYWNrd2FyZFRhYihlKX0pKSxnLl9wYXJzZXIucmVnaXN0ZXJDc2lIYW5kbGVyKHtmaW5hbDoiYCJ9LChmdW5jdGlvbihlKXtyZXR1cm4gZy5jaGFyUG9zQWJzb2x1dGUoZSl9KSksZy5fcGFyc2VyLnJlZ2lzdGVyQ3NpSGFuZGxlcih7ZmluYWw6ImEifSwoZnVuY3Rpb24oZSl7cmV0dXJuIGcuaFBvc2l0aW9uUmVsYXRpdmUoZSl9KSksZy5fcGFyc2VyLnJlZ2lzdGVyQ3NpSGFuZGxlcih7ZmluYWw6ImIifSwoZnVuY3Rpb24oZSl7cmV0dXJuIGcucmVwZWF0UHJlY2VkaW5nQ2hhcmFjdGVyKGUpfSkpLGcuX3BhcnNlci5yZWdpc3RlckNzaUhhbmRsZXIoe2ZpbmFsOiJjIn0sKGZ1bmN0aW9uKGUpe3JldHVybiBnLnNlbmREZXZpY2VBdHRyaWJ1dGVzUHJpbWFyeShlKX0pKSxnLl9wYXJzZXIucmVnaXN0ZXJDc2lIYW5kbGVyKHtwcmVmaXg6Ij4iLGZpbmFsOiJjIn0sKGZ1bmN0aW9uKGUpe3JldHVybiBnLnNlbmREZXZpY2VBdHRyaWJ1dGVzU2Vjb25kYXJ5KGUpfSkpLGcuX3BhcnNlci5yZWdpc3RlckNzaUhhbmRsZXIoe2ZpbmFsOiJkIn0sKGZ1bmN0aW9uKGUpe3JldHVybiBnLmxpbmVQb3NBYnNvbHV0ZShlKX0pKSxnLl9wYXJzZXIucmVnaXN0ZXJDc2lIYW5kbGVyKHtmaW5hbDoiZSJ9LChmdW5jdGlvbihlKXtyZXR1cm4gZy52UG9zaXRpb25SZWxhdGl2ZShlKX0pKSxnLl9wYXJzZXIucmVnaXN0ZXJDc2lIYW5kbGVyKHtmaW5hbDoiZiJ9LChmdW5jdGlvbihlKXtyZXR1cm4gZy5oVlBvc2l0aW9uKGUpfSkpLGcuX3BhcnNlci5yZWdpc3RlckNzaUhhbmRsZXIoe2ZpbmFsOiJnIn0sKGZ1bmN0aW9uKGUpe3JldHVybiBnLnRhYkNsZWFyKGUpfSkpLGcuX3BhcnNlci5yZWdpc3RlckNzaUhhbmRsZXIoe2ZpbmFsOiJoIn0sKGZ1bmN0aW9uKGUpe3JldHVybiBnLnNldE1vZGUoZSl9KSksZy5fcGFyc2VyLnJlZ2lzdGVyQ3NpSGFuZGxlcih7cHJlZml4OiI/IixmaW5hbDoiaCJ9LChmdW5jdGlvbihlKXtyZXR1cm4gZy5zZXRNb2RlUHJpdmF0ZShlKX0pKSxnLl9wYXJzZXIucmVnaXN0ZXJDc2lIYW5kbGVyKHtmaW5hbDoibCJ9LChmdW5jdGlvbihlKXtyZXR1cm4gZy5yZXNldE1vZGUoZSl9KSksZy5fcGFyc2VyLnJlZ2lzdGVyQ3NpSGFuZGxlcih7cHJlZml4OiI/IixmaW5hbDoibCJ9LChmdW5jdGlvbihlKXtyZXR1cm4gZy5yZXNldE1vZGVQcml2YXRlKGUpfSkpLGcuX3BhcnNlci5yZWdpc3RlckNzaUhhbmRsZXIoe2ZpbmFsOiJtIn0sKGZ1bmN0aW9uKGUpe3JldHVybiBnLmNoYXJBdHRyaWJ1dGVzKGUpfSkpLGcuX3BhcnNlci5yZWdpc3RlckNzaUhhbmRsZXIoe2ZpbmFsOiJuIn0sKGZ1bmN0aW9uKGUpe3JldHVybiBnLmRldmljZVN0YXR1cyhlKX0pKSxnLl9wYXJzZXIucmVnaXN0ZXJDc2lIYW5kbGVyKHtwcmVmaXg6Ij8iLGZpbmFsOiJuIn0sKGZ1bmN0aW9uKGUpe3JldHVybiBnLmRldmljZVN0YXR1c1ByaXZhdGUoZSl9KSksZy5fcGFyc2VyLnJlZ2lzdGVyQ3NpSGFuZGxlcih7aW50ZXJtZWRpYXRlczoiISIsZmluYWw6InAifSwoZnVuY3Rpb24oZSl7cmV0dXJuIGcuc29mdFJlc2V0KGUpfSkpLGcuX3BhcnNlci5yZWdpc3RlckNzaUhhbmRsZXIoe2ludGVybWVkaWF0ZXM6IiAiLGZpbmFsOiJxIn0sKGZ1bmN0aW9uKGUpe3JldHVybiBnLnNldEN1cnNvclN0eWxlKGUpfSkpLGcuX3BhcnNlci5yZWdpc3RlckNzaUhhbmRsZXIoe2ZpbmFsOiJyIn0sKGZ1bmN0aW9uKGUpe3JldHVybiBnLnNldFNjcm9sbFJlZ2lvbihlKX0pKSxnLl9wYXJzZXIucmVnaXN0ZXJDc2lIYW5kbGVyKHtmaW5hbDoicyJ9LChmdW5jdGlvbihlKXtyZXR1cm4gZy5zYXZlQ3Vyc29yKGUpfSkpLGcuX3BhcnNlci5yZWdpc3RlckNzaUhhbmRsZXIoe2ZpbmFsOiJ0In0sKGZ1bmN0aW9uKGUpe3JldHVybiBnLndpbmRvd09wdGlvbnMoZSl9KSksZy5fcGFyc2VyLnJlZ2lzdGVyQ3NpSGFuZGxlcih7ZmluYWw6InUifSwoZnVuY3Rpb24oZSl7cmV0dXJuIGcucmVzdG9yZUN1cnNvcihlKX0pKSxnLl9wYXJzZXIucmVnaXN0ZXJDc2lIYW5kbGVyKHtpbnRlcm1lZGlhdGVzOiInIixmaW5hbDoifSJ9LChmdW5jdGlvbihlKXtyZXR1cm4gZy5pbnNlcnRDb2x1bW5zKGUpfSkpLGcuX3BhcnNlci5yZWdpc3RlckNzaUhhbmRsZXIoe2ludGVybWVkaWF0ZXM6IiciLGZpbmFsOiJ+In0sKGZ1bmN0aW9uKGUpe3JldHVybiBnLmRlbGV0ZUNvbHVtbnMoZSl9KSksZy5fcGFyc2VyLnNldEV4ZWN1dGVIYW5kbGVyKHMuQzAuQkVMLChmdW5jdGlvbigpe3JldHVybiBnLmJlbGwoKX0pKSxnLl9wYXJzZXIuc2V0RXhlY3V0ZUhhbmRsZXIocy5DMC5MRiwoZnVuY3Rpb24oKXtyZXR1cm4gZy5saW5lRmVlZCgpfSkpLGcuX3BhcnNlci5zZXRFeGVjdXRlSGFuZGxlcihzLkMwLlZULChmdW5jdGlvbigpe3JldHVybiBnLmxpbmVGZWVkKCl9KSksZy5fcGFyc2VyLnNldEV4ZWN1dGVIYW5kbGVyKHMuQzAuRkYsKGZ1bmN0aW9uKCl7cmV0dXJuIGcubGluZUZlZWQoKX0pKSxnLl9wYXJzZXIuc2V0RXhlY3V0ZUhhbmRsZXIocy5DMC5DUiwoZnVuY3Rpb24oKXtyZXR1cm4gZy5jYXJyaWFnZVJldHVybigpfSkpLGcuX3BhcnNlci5zZXRFeGVjdXRlSGFuZGxlcihzLkMwLkJTLChmdW5jdGlvbigpe3JldHVybiBnLmJhY2tzcGFjZSgpfSkpLGcuX3BhcnNlci5zZXRFeGVjdXRlSGFuZGxlcihzLkMwLkhULChmdW5jdGlvbigpe3JldHVybiBnLnRhYigpfSkpLGcuX3BhcnNlci5zZXRFeGVjdXRlSGFuZGxlcihzLkMwLlNPLChmdW5jdGlvbigpe3JldHVybiBnLnNoaWZ0T3V0KCl9KSksZy5fcGFyc2VyLnNldEV4ZWN1dGVIYW5kbGVyKHMuQzAuU0ksKGZ1bmN0aW9uKCl7cmV0dXJuIGcuc2hpZnRJbigpfSkpLGcuX3BhcnNlci5zZXRFeGVjdXRlSGFuZGxlcihzLkMxLklORCwoZnVuY3Rpb24oKXtyZXR1cm4gZy5pbmRleCgpfSkpLGcuX3BhcnNlci5zZXRFeGVjdXRlSGFuZGxlcihzLkMxLk5FTCwoZnVuY3Rpb24oKXtyZXR1cm4gZy5uZXh0TGluZSgpfSkpLGcuX3BhcnNlci5zZXRFeGVjdXRlSGFuZGxlcihzLkMxLkhUUywoZnVuY3Rpb24oKXtyZXR1cm4gZy50YWJTZXQoKX0pKSxnLl9wYXJzZXIucmVnaXN0ZXJPc2NIYW5kbGVyKDAsbmV3IHkuT3NjSGFuZGxlcigoZnVuY3Rpb24oZSl7cmV0dXJuIGcuc2V0VGl0bGUoZSksZy5zZXRJY29uTmFtZShlKSwhMH0pKSksZy5fcGFyc2VyLnJlZ2lzdGVyT3NjSGFuZGxlcigxLG5ldyB5Lk9zY0hhbmRsZXIoKGZ1bmN0aW9uKGUpe3JldHVybiBnLnNldEljb25OYW1lKGUpfSkpKSxnLl9wYXJzZXIucmVnaXN0ZXJPc2NIYW5kbGVyKDIsbmV3IHkuT3NjSGFuZGxlcigoZnVuY3Rpb24oZSl7cmV0dXJuIGcuc2V0VGl0bGUoZSl9KSkpLGcuX3BhcnNlci5yZWdpc3Rlck9zY0hhbmRsZXIoNCxuZXcgeS5Pc2NIYW5kbGVyKChmdW5jdGlvbihlKXtyZXR1cm4gZy5zZXRPclJlcG9ydEluZGV4ZWRDb2xvcihlKX0pKSksZy5fcGFyc2VyLnJlZ2lzdGVyT3NjSGFuZGxlcigxMCxuZXcgeS5Pc2NIYW5kbGVyKChmdW5jdGlvbihlKXtyZXR1cm4gZy5zZXRPclJlcG9ydEZnQ29sb3IoZSl9KSkpLGcuX3BhcnNlci5yZWdpc3Rlck9zY0hhbmRsZXIoMTEsbmV3IHkuT3NjSGFuZGxlcigoZnVuY3Rpb24oZSl7cmV0dXJuIGcuc2V0T3JSZXBvcnRCZ0NvbG9yKGUpfSkpKSxnLl9wYXJzZXIucmVnaXN0ZXJPc2NIYW5kbGVyKDEyLG5ldyB5Lk9zY0hhbmRsZXIoKGZ1bmN0aW9uKGUpe3JldHVybiBnLnNldE9yUmVwb3J0Q3Vyc29yQ29sb3IoZSl9KSkpLGcuX3BhcnNlci5yZWdpc3Rlck9zY0hhbmRsZXIoMTA0LG5ldyB5Lk9zY0hhbmRsZXIoKGZ1bmN0aW9uKGUpe3JldHVybiBnLnJlc3RvcmVJbmRleGVkQ29sb3IoZSl9KSkpLGcuX3BhcnNlci5yZWdpc3Rlck9zY0hhbmRsZXIoMTEwLG5ldyB5Lk9zY0hhbmRsZXIoKGZ1bmN0aW9uKGUpe3JldHVybiBnLnJlc3RvcmVGZ0NvbG9yKGUpfSkpKSxnLl9wYXJzZXIucmVnaXN0ZXJPc2NIYW5kbGVyKDExMSxuZXcgeS5Pc2NIYW5kbGVyKChmdW5jdGlvbihlKXtyZXR1cm4gZy5yZXN0b3JlQmdDb2xvcihlKX0pKSksZy5fcGFyc2VyLnJlZ2lzdGVyT3NjSGFuZGxlcigxMTIsbmV3IHkuT3NjSGFuZGxlcigoZnVuY3Rpb24oZSl7cmV0dXJuIGcucmVzdG9yZUN1cnNvckNvbG9yKGUpfSkpKSxnLl9wYXJzZXIucmVnaXN0ZXJFc2NIYW5kbGVyKHtmaW5hbDoiNyJ9LChmdW5jdGlvbigpe3JldHVybiBnLnNhdmVDdXJzb3IoKX0pKSxnLl9wYXJzZXIucmVnaXN0ZXJFc2NIYW5kbGVyKHtmaW5hbDoiOCJ9LChmdW5jdGlvbigpe3JldHVybiBnLnJlc3RvcmVDdXJzb3IoKX0pKSxnLl9wYXJzZXIucmVnaXN0ZXJFc2NIYW5kbGVyKHtmaW5hbDoiRCJ9LChmdW5jdGlvbigpe3JldHVybiBnLmluZGV4KCl9KSksZy5fcGFyc2VyLnJlZ2lzdGVyRXNjSGFuZGxlcih7ZmluYWw6IkUifSwoZnVuY3Rpb24oKXtyZXR1cm4gZy5uZXh0TGluZSgpfSkpLGcuX3BhcnNlci5yZWdpc3RlckVzY0hhbmRsZXIoe2ZpbmFsOiJIIn0sKGZ1bmN0aW9uKCl7cmV0dXJuIGcudGFiU2V0KCl9KSksZy5fcGFyc2VyLnJlZ2lzdGVyRXNjSGFuZGxlcih7ZmluYWw6Ik0ifSwoZnVuY3Rpb24oKXtyZXR1cm4gZy5yZXZlcnNlSW5kZXgoKX0pKSxnLl9wYXJzZXIucmVnaXN0ZXJFc2NIYW5kbGVyKHtmaW5hbDoiPSJ9LChmdW5jdGlvbigpe3JldHVybiBnLmtleXBhZEFwcGxpY2F0aW9uTW9kZSgpfSkpLGcuX3BhcnNlci5yZWdpc3RlckVzY0hhbmRsZXIoe2ZpbmFsOiI+In0sKGZ1bmN0aW9uKCl7cmV0dXJuIGcua2V5cGFkTnVtZXJpY01vZGUoKX0pKSxnLl9wYXJzZXIucmVnaXN0ZXJFc2NIYW5kbGVyKHtmaW5hbDoiYyJ9LChmdW5jdGlvbigpe3JldHVybiBnLmZ1bGxSZXNldCgpfSkpLGcuX3BhcnNlci5yZWdpc3RlckVzY0hhbmRsZXIoe2ZpbmFsOiJuIn0sKGZ1bmN0aW9uKCl7cmV0dXJuIGcuc2V0Z0xldmVsKDIpfSkpLGcuX3BhcnNlci5yZWdpc3RlckVzY0hhbmRsZXIoe2ZpbmFsOiJvIn0sKGZ1bmN0aW9uKCl7cmV0dXJuIGcuc2V0Z0xldmVsKDMpfSkpLGcuX3BhcnNlci5yZWdpc3RlckVzY0hhbmRsZXIoe2ZpbmFsOiJ8In0sKGZ1bmN0aW9uKCl7cmV0dXJuIGcuc2V0Z0xldmVsKDMpfSkpLGcuX3BhcnNlci5yZWdpc3RlckVzY0hhbmRsZXIoe2ZpbmFsOiJ9In0sKGZ1bmN0aW9uKCl7cmV0dXJuIGcuc2V0Z0xldmVsKDIpfSkpLGcuX3BhcnNlci5yZWdpc3RlckVzY0hhbmRsZXIoe2ZpbmFsOiJ+In0sKGZ1bmN0aW9uKCl7cmV0dXJuIGcuc2V0Z0xldmVsKDEpfSkpLGcuX3BhcnNlci5yZWdpc3RlckVzY0hhbmRsZXIoe2ludGVybWVkaWF0ZXM6IiUiLGZpbmFsOiJAIn0sKGZ1bmN0aW9uKCl7cmV0dXJuIGcuc2VsZWN0RGVmYXVsdENoYXJzZXQoKX0pKSxnLl9wYXJzZXIucmVnaXN0ZXJFc2NIYW5kbGVyKHtpbnRlcm1lZGlhdGVzOiIlIixmaW5hbDoiRyJ9LChmdW5jdGlvbigpe3JldHVybiBnLnNlbGVjdERlZmF1bHRDaGFyc2V0KCl9KSk7dmFyIG09ZnVuY3Rpb24oZSl7Yi5fcGFyc2VyLnJlZ2lzdGVyRXNjSGFuZGxlcih7aW50ZXJtZWRpYXRlczoiKCIsZmluYWw6ZX0sKGZ1bmN0aW9uKCl7cmV0dXJuIGcuc2VsZWN0Q2hhcnNldCgiKCIrZSl9KSksYi5fcGFyc2VyLnJlZ2lzdGVyRXNjSGFuZGxlcih7aW50ZXJtZWRpYXRlczoiKSIsZmluYWw6ZX0sKGZ1bmN0aW9uKCl7cmV0dXJuIGcuc2VsZWN0Q2hhcnNldCgiKSIrZSl9KSksYi5fcGFyc2VyLnJlZ2lzdGVyRXNjSGFuZGxlcih7aW50ZXJtZWRpYXRlczoiKiIsZmluYWw6ZX0sKGZ1bmN0aW9uKCl7cmV0dXJuIGcuc2VsZWN0Q2hhcnNldCgiKiIrZSl9KSksYi5fcGFyc2VyLnJlZ2lzdGVyRXNjSGFuZGxlcih7aW50ZXJtZWRpYXRlczoiKyIsZmluYWw6ZX0sKGZ1bmN0aW9uKCl7cmV0dXJuIGcuc2VsZWN0Q2hhcnNldCgiKyIrZSl9KSksYi5fcGFyc2VyLnJlZ2lzdGVyRXNjSGFuZGxlcih7aW50ZXJtZWRpYXRlczoiLSIsZmluYWw6ZX0sKGZ1bmN0aW9uKCl7cmV0dXJuIGcuc2VsZWN0Q2hhcnNldCgiLSIrZSl9KSksYi5fcGFyc2VyLnJlZ2lzdGVyRXNjSGFuZGxlcih7aW50ZXJtZWRpYXRlczoiLiIsZmluYWw6ZX0sKGZ1bmN0aW9uKCl7cmV0dXJuIGcuc2VsZWN0Q2hhcnNldCgiLiIrZSl9KSksYi5fcGFyc2VyLnJlZ2lzdGVyRXNjSGFuZGxlcih7aW50ZXJtZWRpYXRlczoiLyIsZmluYWw6ZX0sKGZ1bmN0aW9uKCl7cmV0dXJuIGcuc2VsZWN0Q2hhcnNldCgiLyIrZSl9KSl9LGI9dGhpcztmb3IodmFyIFMgaW4gYS5DSEFSU0VUUyltKFMpO3JldHVybiBnLl9wYXJzZXIucmVnaXN0ZXJFc2NIYW5kbGVyKHtpbnRlcm1lZGlhdGVzOiIjIixmaW5hbDoiOCJ9LChmdW5jdGlvbigpe3JldHVybiBnLnNjcmVlbkFsaWdubWVudFBhdHRlcm4oKX0pKSxnLl9wYXJzZXIuc2V0RXJyb3JIYW5kbGVyKChmdW5jdGlvbihlKXtyZXR1cm4gZy5fbG9nU2VydmljZS5lcnJvcigiUGFyc2luZyBlcnJvcjogIixlKSxlfSkpLGcuX3BhcnNlci5yZWdpc3RlckRjc0hhbmRsZXIoe2ludGVybWVkaWF0ZXM6IiQiLGZpbmFsOiJxIn0sbmV3IEwoZy5fYnVmZmVyU2VydmljZSxnLl9jb3JlU2VydmljZSxnLl9sb2dTZXJ2aWNlLGcuX29wdGlvbnNTZXJ2aWNlKSksZ31yZXR1cm4gbih0LGUpLE9iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LnByb3RvdHlwZSwib25SZXF1ZXN0QmVsbCIse2dldDpmdW5jdGlvbigpe3JldHVybiB0aGlzLl9vblJlcXVlc3RCZWxsLmV2ZW50fSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLE9iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LnByb3RvdHlwZSwib25SZXF1ZXN0UmVmcmVzaFJvd3MiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fb25SZXF1ZXN0UmVmcmVzaFJvd3MuZXZlbnR9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KHQucHJvdG90eXBlLCJvblJlcXVlc3RSZXNldCIse2dldDpmdW5jdGlvbigpe3JldHVybiB0aGlzLl9vblJlcXVlc3RSZXNldC5ldmVudH0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxPYmplY3QuZGVmaW5lUHJvcGVydHkodC5wcm90b3R5cGUsIm9uUmVxdWVzdFNlbmRGb2N1cyIse2dldDpmdW5jdGlvbigpe3JldHVybiB0aGlzLl9vblJlcXVlc3RTZW5kRm9jdXMuZXZlbnR9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KHQucHJvdG90eXBlLCJvblJlcXVlc3RTeW5jU2Nyb2xsQmFyIix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX29uUmVxdWVzdFN5bmNTY3JvbGxCYXIuZXZlbnR9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KHQucHJvdG90eXBlLCJvblJlcXVlc3RXaW5kb3dzT3B0aW9uc1JlcG9ydCIse2dldDpmdW5jdGlvbigpe3JldHVybiB0aGlzLl9vblJlcXVlc3RXaW5kb3dzT3B0aW9uc1JlcG9ydC5ldmVudH0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxPYmplY3QuZGVmaW5lUHJvcGVydHkodC5wcm90b3R5cGUsIm9uQTExeUNoYXIiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fb25BMTF5Q2hhci5ldmVudH0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxPYmplY3QuZGVmaW5lUHJvcGVydHkodC5wcm90b3R5cGUsIm9uQTExeVRhYiIse2dldDpmdW5jdGlvbigpe3JldHVybiB0aGlzLl9vbkExMXlUYWIuZXZlbnR9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KHQucHJvdG90eXBlLCJvbkN1cnNvck1vdmUiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fb25DdXJzb3JNb3ZlLmV2ZW50fSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLE9iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LnByb3RvdHlwZSwib25MaW5lRmVlZCIse2dldDpmdW5jdGlvbigpe3JldHVybiB0aGlzLl9vbkxpbmVGZWVkLmV2ZW50fSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLE9iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LnByb3RvdHlwZSwib25TY3JvbGwiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fb25TY3JvbGwuZXZlbnR9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KHQucHJvdG90eXBlLCJvblRpdGxlQ2hhbmdlIix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX29uVGl0bGVDaGFuZ2UuZXZlbnR9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KHQucHJvdG90eXBlLCJvbkNvbG9yIix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX29uQ29sb3IuZXZlbnR9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksdC5wcm90b3R5cGUuZGlzcG9zZT1mdW5jdGlvbigpe2UucHJvdG90eXBlLmRpc3Bvc2UuY2FsbCh0aGlzKX0sdC5wcm90b3R5cGUuX3ByZXNlcnZlU3RhY2s9ZnVuY3Rpb24oZSx0LHIsaSl7dGhpcy5fcGFyc2VTdGFjay5wYXVzZWQ9ITAsdGhpcy5fcGFyc2VTdGFjay5jdXJzb3JTdGFydFg9ZSx0aGlzLl9wYXJzZVN0YWNrLmN1cnNvclN0YXJ0WT10LHRoaXMuX3BhcnNlU3RhY2suZGVjb2RlZExlbmd0aD1yLHRoaXMuX3BhcnNlU3RhY2sucG9zaXRpb249aX0sdC5wcm90b3R5cGUuX2xvZ1Nsb3dSZXNvbHZpbmdBc3luYz1mdW5jdGlvbihlKXt0aGlzLl9sb2dTZXJ2aWNlLmxvZ0xldmVsPD1nLkxvZ0xldmVsRW51bS5XQVJOJiZQcm9taXNlLnJhY2UoW2UsbmV3IFByb21pc2UoKGZ1bmN0aW9uKGUsdCl7cmV0dXJuIHNldFRpbWVvdXQoKGZ1bmN0aW9uKCl7cmV0dXJuIHQoIiNTTE9XX1RJTUVPVVQiKX0pLDVlMyl9KSldKS5jYXRjaCgoZnVuY3Rpb24oZSl7aWYoIiNTTE9XX1RJTUVPVVQiIT09ZSl0aHJvdyBlO2NvbnNvbGUud2FybigiYXN5bmMgcGFyc2VyIGhhbmRsZXIgdGFraW5nIGxvbmdlciB0aGFuIDUwMDAgbXMiKX0pKX0sdC5wcm90b3R5cGUucGFyc2U9ZnVuY3Rpb24oZSx0KXt2YXIgcixpPXRoaXMuX2FjdGl2ZUJ1ZmZlci54LG49dGhpcy5fYWN0aXZlQnVmZmVyLnksbz0wLHM9dGhpcy5fcGFyc2VTdGFjay5wYXVzZWQ7aWYocyl7aWYocj10aGlzLl9wYXJzZXIucGFyc2UodGhpcy5fcGFyc2VCdWZmZXIsdGhpcy5fcGFyc2VTdGFjay5kZWNvZGVkTGVuZ3RoLHQpKXJldHVybiB0aGlzLl9sb2dTbG93UmVzb2x2aW5nQXN5bmMocikscjtpPXRoaXMuX3BhcnNlU3RhY2suY3Vyc29yU3RhcnRYLG49dGhpcy5fcGFyc2VTdGFjay5jdXJzb3JTdGFydFksdGhpcy5fcGFyc2VTdGFjay5wYXVzZWQ9ITEsZS5sZW5ndGg+QyYmKG89dGhpcy5fcGFyc2VTdGFjay5wb3NpdGlvbitDKX1pZih0aGlzLl9sb2dTZXJ2aWNlLmxvZ0xldmVsPD1nLkxvZ0xldmVsRW51bS5ERUJVRyYmdGhpcy5fbG9nU2VydmljZS5kZWJ1ZygicGFyc2luZyBkYXRhIisoInN0cmluZyI9PXR5cGVvZiBlPycgIicrZSsnIic6IiIpLCJzdHJpbmciPT10eXBlb2YgZT9lLnNwbGl0KCIiKS5tYXAoKGZ1bmN0aW9uKGUpe3JldHVybiBlLmNoYXJDb2RlQXQoMCl9KSk6ZSksdGhpcy5fcGFyc2VCdWZmZXIubGVuZ3RoPGUubGVuZ3RoJiZ0aGlzLl9wYXJzZUJ1ZmZlci5sZW5ndGg8QyYmKHRoaXMuX3BhcnNlQnVmZmVyPW5ldyBVaW50MzJBcnJheShNYXRoLm1pbihlLmxlbmd0aCxDKSkpLHN8fHRoaXMuX2RpcnR5Um93U2VydmljZS5jbGVhclJhbmdlKCksZS5sZW5ndGg+Qylmb3IodmFyIGE9bzthPGUubGVuZ3RoO2ErPUMpe3ZhciBjPWErQzxlLmxlbmd0aD9hK0M6ZS5sZW5ndGgsbD0ic3RyaW5nIj09dHlwZW9mIGU/dGhpcy5fc3RyaW5nRGVjb2Rlci5kZWNvZGUoZS5zdWJzdHJpbmcoYSxjKSx0aGlzLl9wYXJzZUJ1ZmZlcik6dGhpcy5fdXRmOERlY29kZXIuZGVjb2RlKGUuc3ViYXJyYXkoYSxjKSx0aGlzLl9wYXJzZUJ1ZmZlcik7aWYocj10aGlzLl9wYXJzZXIucGFyc2UodGhpcy5fcGFyc2VCdWZmZXIsbCkpcmV0dXJuIHRoaXMuX3ByZXNlcnZlU3RhY2soaSxuLGwsYSksdGhpcy5fbG9nU2xvd1Jlc29sdmluZ0FzeW5jKHIpLHJ9ZWxzZSBpZighcyYmKGw9InN0cmluZyI9PXR5cGVvZiBlP3RoaXMuX3N0cmluZ0RlY29kZXIuZGVjb2RlKGUsdGhpcy5fcGFyc2VCdWZmZXIpOnRoaXMuX3V0ZjhEZWNvZGVyLmRlY29kZShlLHRoaXMuX3BhcnNlQnVmZmVyKSxyPXRoaXMuX3BhcnNlci5wYXJzZSh0aGlzLl9wYXJzZUJ1ZmZlcixsKSkpcmV0dXJuIHRoaXMuX3ByZXNlcnZlU3RhY2soaSxuLGwsMCksdGhpcy5fbG9nU2xvd1Jlc29sdmluZ0FzeW5jKHIpLHI7dGhpcy5fYWN0aXZlQnVmZmVyLng9PT1pJiZ0aGlzLl9hY3RpdmVCdWZmZXIueT09PW58fHRoaXMuX29uQ3Vyc29yTW92ZS5maXJlKCksdGhpcy5fb25SZXF1ZXN0UmVmcmVzaFJvd3MuZmlyZSh0aGlzLl9kaXJ0eVJvd1NlcnZpY2Uuc3RhcnQsdGhpcy5fZGlydHlSb3dTZXJ2aWNlLmVuZCl9LHQucHJvdG90eXBlLnByaW50PWZ1bmN0aW9uKGUsdCxyKXt2YXIgaSxuLG89dGhpcy5fY2hhcnNldFNlcnZpY2UuY2hhcnNldCxzPXRoaXMuX29wdGlvbnNTZXJ2aWNlLm9wdGlvbnMuc2NyZWVuUmVhZGVyTW9kZSxhPXRoaXMuX2J1ZmZlclNlcnZpY2UuY29scyxjPXRoaXMuX2NvcmVTZXJ2aWNlLmRlY1ByaXZhdGVNb2Rlcy53cmFwYXJvdW5kLGw9dGhpcy5fY29yZVNlcnZpY2UubW9kZXMuaW5zZXJ0TW9kZSx1PXRoaXMuX2N1ckF0dHJEYXRhLGY9dGhpcy5fYWN0aXZlQnVmZmVyLmxpbmVzLmdldCh0aGlzLl9hY3RpdmVCdWZmZXIueWJhc2UrdGhpcy5fYWN0aXZlQnVmZmVyLnkpO3RoaXMuX2RpcnR5Um93U2VydmljZS5tYXJrRGlydHkodGhpcy5fYWN0aXZlQnVmZmVyLnkpLHRoaXMuX2FjdGl2ZUJ1ZmZlci54JiZyLXQ+MCYmMj09PWYuZ2V0V2lkdGgodGhpcy5fYWN0aXZlQnVmZmVyLngtMSkmJmYuc2V0Q2VsbEZyb21Db2RlUG9pbnQodGhpcy5fYWN0aXZlQnVmZmVyLngtMSwwLDEsdS5mZyx1LmJnLHUuZXh0ZW5kZWQpO2Zvcih2YXIgXz10O188cjsrK18pe2lmKGk9ZVtfXSxuPXRoaXMuX3VuaWNvZGVTZXJ2aWNlLndjd2lkdGgoaSksaTwxMjcmJm8pe3ZhciBwPW9bU3RyaW5nLmZyb21DaGFyQ29kZShpKV07cCYmKGk9cC5jaGFyQ29kZUF0KDApKX1pZihzJiZ0aGlzLl9vbkExMXlDaGFyLmZpcmUoKDAsaC5zdHJpbmdGcm9tQ29kZVBvaW50KShpKSksbnx8IXRoaXMuX2FjdGl2ZUJ1ZmZlci54KXtpZih0aGlzLl9hY3RpdmVCdWZmZXIueCtuLTE+PWEpaWYoYyl7Zm9yKDt0aGlzLl9hY3RpdmVCdWZmZXIueDxhOylmLnNldENlbGxGcm9tQ29kZVBvaW50KHRoaXMuX2FjdGl2ZUJ1ZmZlci54KyssMCwxLHUuZmcsdS5iZyx1LmV4dGVuZGVkKTt0aGlzLl9hY3RpdmVCdWZmZXIueD0wLHRoaXMuX2FjdGl2ZUJ1ZmZlci55KyssdGhpcy5fYWN0aXZlQnVmZmVyLnk9PT10aGlzLl9hY3RpdmVCdWZmZXIuc2Nyb2xsQm90dG9tKzE/KHRoaXMuX2FjdGl2ZUJ1ZmZlci55LS0sdGhpcy5fYnVmZmVyU2VydmljZS5zY3JvbGwodGhpcy5fZXJhc2VBdHRyRGF0YSgpLCEwKSk6KHRoaXMuX2FjdGl2ZUJ1ZmZlci55Pj10aGlzLl9idWZmZXJTZXJ2aWNlLnJvd3MmJih0aGlzLl9hY3RpdmVCdWZmZXIueT10aGlzLl9idWZmZXJTZXJ2aWNlLnJvd3MtMSksdGhpcy5fYWN0aXZlQnVmZmVyLmxpbmVzLmdldCh0aGlzLl9hY3RpdmVCdWZmZXIueWJhc2UrdGhpcy5fYWN0aXZlQnVmZmVyLnkpLmlzV3JhcHBlZD0hMCksZj10aGlzLl9hY3RpdmVCdWZmZXIubGluZXMuZ2V0KHRoaXMuX2FjdGl2ZUJ1ZmZlci55YmFzZSt0aGlzLl9hY3RpdmVCdWZmZXIueSl9ZWxzZSBpZih0aGlzLl9hY3RpdmVCdWZmZXIueD1hLTEsMj09PW4pY29udGludWU7aWYobCYmKGYuaW5zZXJ0Q2VsbHModGhpcy5fYWN0aXZlQnVmZmVyLngsbix0aGlzLl9hY3RpdmVCdWZmZXIuZ2V0TnVsbENlbGwodSksdSksMj09PWYuZ2V0V2lkdGgoYS0xKSYmZi5zZXRDZWxsRnJvbUNvZGVQb2ludChhLTEsZC5OVUxMX0NFTExfQ09ERSxkLk5VTExfQ0VMTF9XSURUSCx1LmZnLHUuYmcsdS5leHRlbmRlZCkpLGYuc2V0Q2VsbEZyb21Db2RlUG9pbnQodGhpcy5fYWN0aXZlQnVmZmVyLngrKyxpLG4sdS5mZyx1LmJnLHUuZXh0ZW5kZWQpLG4+MClmb3IoOy0tbjspZi5zZXRDZWxsRnJvbUNvZGVQb2ludCh0aGlzLl9hY3RpdmVCdWZmZXIueCsrLDAsMCx1LmZnLHUuYmcsdS5leHRlbmRlZCl9ZWxzZSBmLmdldFdpZHRoKHRoaXMuX2FjdGl2ZUJ1ZmZlci54LTEpP2YuYWRkQ29kZXBvaW50VG9DZWxsKHRoaXMuX2FjdGl2ZUJ1ZmZlci54LTEsaSk6Zi5hZGRDb2RlcG9pbnRUb0NlbGwodGhpcy5fYWN0aXZlQnVmZmVyLngtMixpKX1yLXQ+MCYmKGYubG9hZENlbGwodGhpcy5fYWN0aXZlQnVmZmVyLngtMSx0aGlzLl93b3JrQ2VsbCksMj09PXRoaXMuX3dvcmtDZWxsLmdldFdpZHRoKCl8fHRoaXMuX3dvcmtDZWxsLmdldENvZGUoKT42NTUzNT90aGlzLl9wYXJzZXIucHJlY2VkaW5nQ29kZXBvaW50PTA6dGhpcy5fd29ya0NlbGwuaXNDb21iaW5lZCgpP3RoaXMuX3BhcnNlci5wcmVjZWRpbmdDb2RlcG9pbnQ9dGhpcy5fd29ya0NlbGwuZ2V0Q2hhcnMoKS5jaGFyQ29kZUF0KDApOnRoaXMuX3BhcnNlci5wcmVjZWRpbmdDb2RlcG9pbnQ9dGhpcy5fd29ya0NlbGwuY29udGVudCksdGhpcy5fYWN0aXZlQnVmZmVyLng8YSYmci10PjAmJjA9PT1mLmdldFdpZHRoKHRoaXMuX2FjdGl2ZUJ1ZmZlci54KSYmIWYuaGFzQ29udGVudCh0aGlzLl9hY3RpdmVCdWZmZXIueCkmJmYuc2V0Q2VsbEZyb21Db2RlUG9pbnQodGhpcy5fYWN0aXZlQnVmZmVyLngsMCwxLHUuZmcsdS5iZyx1LmV4dGVuZGVkKSx0aGlzLl9kaXJ0eVJvd1NlcnZpY2UubWFya0RpcnR5KHRoaXMuX2FjdGl2ZUJ1ZmZlci55KX0sdC5wcm90b3R5cGUucmVnaXN0ZXJDc2lIYW5kbGVyPWZ1bmN0aW9uKGUsdCl7dmFyIHI9dGhpcztyZXR1cm4idCIhPT1lLmZpbmFsfHxlLnByZWZpeHx8ZS5pbnRlcm1lZGlhdGVzP3RoaXMuX3BhcnNlci5yZWdpc3RlckNzaUhhbmRsZXIoZSx0KTp0aGlzLl9wYXJzZXIucmVnaXN0ZXJDc2lIYW5kbGVyKGUsKGZ1bmN0aW9uKGUpe3JldHVybiF3KGUucGFyYW1zWzBdLHIuX29wdGlvbnNTZXJ2aWNlLm9wdGlvbnMud2luZG93T3B0aW9ucyl8fHQoZSl9KSl9LHQucHJvdG90eXBlLnJlZ2lzdGVyRGNzSGFuZGxlcj1mdW5jdGlvbihlLHQpe3JldHVybiB0aGlzLl9wYXJzZXIucmVnaXN0ZXJEY3NIYW5kbGVyKGUsbmV3IG0uRGNzSGFuZGxlcih0KSl9LHQucHJvdG90eXBlLnJlZ2lzdGVyRXNjSGFuZGxlcj1mdW5jdGlvbihlLHQpe3JldHVybiB0aGlzLl9wYXJzZXIucmVnaXN0ZXJFc2NIYW5kbGVyKGUsdCl9LHQucHJvdG90eXBlLnJlZ2lzdGVyT3NjSGFuZGxlcj1mdW5jdGlvbihlLHQpe3JldHVybiB0aGlzLl9wYXJzZXIucmVnaXN0ZXJPc2NIYW5kbGVyKGUsbmV3IHkuT3NjSGFuZGxlcih0KSl9LHQucHJvdG90eXBlLmJlbGw9ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fb25SZXF1ZXN0QmVsbC5maXJlKCksITB9LHQucHJvdG90eXBlLmxpbmVGZWVkPWZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX2RpcnR5Um93U2VydmljZS5tYXJrRGlydHkodGhpcy5fYWN0aXZlQnVmZmVyLnkpLHRoaXMuX29wdGlvbnNTZXJ2aWNlLm9wdGlvbnMuY29udmVydEVvbCYmKHRoaXMuX2FjdGl2ZUJ1ZmZlci54PTApLHRoaXMuX2FjdGl2ZUJ1ZmZlci55KyssdGhpcy5fYWN0aXZlQnVmZmVyLnk9PT10aGlzLl9hY3RpdmVCdWZmZXIuc2Nyb2xsQm90dG9tKzE/KHRoaXMuX2FjdGl2ZUJ1ZmZlci55LS0sdGhpcy5fYnVmZmVyU2VydmljZS5zY3JvbGwodGhpcy5fZXJhc2VBdHRyRGF0YSgpKSk6dGhpcy5fYWN0aXZlQnVmZmVyLnk+PXRoaXMuX2J1ZmZlclNlcnZpY2Uucm93cyYmKHRoaXMuX2FjdGl2ZUJ1ZmZlci55PXRoaXMuX2J1ZmZlclNlcnZpY2Uucm93cy0xKSx0aGlzLl9hY3RpdmVCdWZmZXIueD49dGhpcy5fYnVmZmVyU2VydmljZS5jb2xzJiZ0aGlzLl9hY3RpdmVCdWZmZXIueC0tLHRoaXMuX2RpcnR5Um93U2VydmljZS5tYXJrRGlydHkodGhpcy5fYWN0aXZlQnVmZmVyLnkpLHRoaXMuX29uTGluZUZlZWQuZmlyZSgpLCEwfSx0LnByb3RvdHlwZS5jYXJyaWFnZVJldHVybj1mdW5jdGlvbigpe3JldHVybiB0aGlzLl9hY3RpdmVCdWZmZXIueD0wLCEwfSx0LnByb3RvdHlwZS5iYWNrc3BhY2U9ZnVuY3Rpb24oKXt2YXIgZTtpZighdGhpcy5fY29yZVNlcnZpY2UuZGVjUHJpdmF0ZU1vZGVzLnJldmVyc2VXcmFwYXJvdW5kKXJldHVybiB0aGlzLl9yZXN0cmljdEN1cnNvcigpLHRoaXMuX2FjdGl2ZUJ1ZmZlci54PjAmJnRoaXMuX2FjdGl2ZUJ1ZmZlci54LS0sITA7aWYodGhpcy5fcmVzdHJpY3RDdXJzb3IodGhpcy5fYnVmZmVyU2VydmljZS5jb2xzKSx0aGlzLl9hY3RpdmVCdWZmZXIueD4wKXRoaXMuX2FjdGl2ZUJ1ZmZlci54LS07ZWxzZSBpZigwPT09dGhpcy5fYWN0aXZlQnVmZmVyLngmJnRoaXMuX2FjdGl2ZUJ1ZmZlci55PnRoaXMuX2FjdGl2ZUJ1ZmZlci5zY3JvbGxUb3AmJnRoaXMuX2FjdGl2ZUJ1ZmZlci55PD10aGlzLl9hY3RpdmVCdWZmZXIuc2Nyb2xsQm90dG9tJiYobnVsbD09PShlPXRoaXMuX2FjdGl2ZUJ1ZmZlci5saW5lcy5nZXQodGhpcy5fYWN0aXZlQnVmZmVyLnliYXNlK3RoaXMuX2FjdGl2ZUJ1ZmZlci55KSl8fHZvaWQgMD09PWU/dm9pZCAwOmUuaXNXcmFwcGVkKSl7dGhpcy5fYWN0aXZlQnVmZmVyLmxpbmVzLmdldCh0aGlzLl9hY3RpdmVCdWZmZXIueWJhc2UrdGhpcy5fYWN0aXZlQnVmZmVyLnkpLmlzV3JhcHBlZD0hMSx0aGlzLl9hY3RpdmVCdWZmZXIueS0tLHRoaXMuX2FjdGl2ZUJ1ZmZlci54PXRoaXMuX2J1ZmZlclNlcnZpY2UuY29scy0xO3ZhciB0PXRoaXMuX2FjdGl2ZUJ1ZmZlci5saW5lcy5nZXQodGhpcy5fYWN0aXZlQnVmZmVyLnliYXNlK3RoaXMuX2FjdGl2ZUJ1ZmZlci55KTt0Lmhhc1dpZHRoKHRoaXMuX2FjdGl2ZUJ1ZmZlci54KSYmIXQuaGFzQ29udGVudCh0aGlzLl9hY3RpdmVCdWZmZXIueCkmJnRoaXMuX2FjdGl2ZUJ1ZmZlci54LS19cmV0dXJuIHRoaXMuX3Jlc3RyaWN0Q3Vyc29yKCksITB9LHQucHJvdG90eXBlLnRhYj1mdW5jdGlvbigpe2lmKHRoaXMuX2FjdGl2ZUJ1ZmZlci54Pj10aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHMpcmV0dXJuITA7dmFyIGU9dGhpcy5fYWN0aXZlQnVmZmVyLng7cmV0dXJuIHRoaXMuX2FjdGl2ZUJ1ZmZlci54PXRoaXMuX2FjdGl2ZUJ1ZmZlci5uZXh0U3RvcCgpLHRoaXMuX29wdGlvbnNTZXJ2aWNlLm9wdGlvbnMuc2NyZWVuUmVhZGVyTW9kZSYmdGhpcy5fb25BMTF5VGFiLmZpcmUodGhpcy5fYWN0aXZlQnVmZmVyLngtZSksITB9LHQucHJvdG90eXBlLnNoaWZ0T3V0PWZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX2NoYXJzZXRTZXJ2aWNlLnNldGdMZXZlbCgxKSwhMH0sdC5wcm90b3R5cGUuc2hpZnRJbj1mdW5jdGlvbigpe3JldHVybiB0aGlzLl9jaGFyc2V0U2VydmljZS5zZXRnTGV2ZWwoMCksITB9LHQucHJvdG90eXBlLl9yZXN0cmljdEN1cnNvcj1mdW5jdGlvbihlKXt2b2lkIDA9PT1lJiYoZT10aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHMtMSksdGhpcy5fYWN0aXZlQnVmZmVyLng9TWF0aC5taW4oZSxNYXRoLm1heCgwLHRoaXMuX2FjdGl2ZUJ1ZmZlci54KSksdGhpcy5fYWN0aXZlQnVmZmVyLnk9dGhpcy5fY29yZVNlcnZpY2UuZGVjUHJpdmF0ZU1vZGVzLm9yaWdpbj9NYXRoLm1pbih0aGlzLl9hY3RpdmVCdWZmZXIuc2Nyb2xsQm90dG9tLE1hdGgubWF4KHRoaXMuX2FjdGl2ZUJ1ZmZlci5zY3JvbGxUb3AsdGhpcy5fYWN0aXZlQnVmZmVyLnkpKTpNYXRoLm1pbih0aGlzLl9idWZmZXJTZXJ2aWNlLnJvd3MtMSxNYXRoLm1heCgwLHRoaXMuX2FjdGl2ZUJ1ZmZlci55KSksdGhpcy5fZGlydHlSb3dTZXJ2aWNlLm1hcmtEaXJ0eSh0aGlzLl9hY3RpdmVCdWZmZXIueSl9LHQucHJvdG90eXBlLl9zZXRDdXJzb3I9ZnVuY3Rpb24oZSx0KXt0aGlzLl9kaXJ0eVJvd1NlcnZpY2UubWFya0RpcnR5KHRoaXMuX2FjdGl2ZUJ1ZmZlci55KSx0aGlzLl9jb3JlU2VydmljZS5kZWNQcml2YXRlTW9kZXMub3JpZ2luPyh0aGlzLl9hY3RpdmVCdWZmZXIueD1lLHRoaXMuX2FjdGl2ZUJ1ZmZlci55PXRoaXMuX2FjdGl2ZUJ1ZmZlci5zY3JvbGxUb3ArdCk6KHRoaXMuX2FjdGl2ZUJ1ZmZlci54PWUsdGhpcy5fYWN0aXZlQnVmZmVyLnk9dCksdGhpcy5fcmVzdHJpY3RDdXJzb3IoKSx0aGlzLl9kaXJ0eVJvd1NlcnZpY2UubWFya0RpcnR5KHRoaXMuX2FjdGl2ZUJ1ZmZlci55KX0sdC5wcm90b3R5cGUuX21vdmVDdXJzb3I9ZnVuY3Rpb24oZSx0KXt0aGlzLl9yZXN0cmljdEN1cnNvcigpLHRoaXMuX3NldEN1cnNvcih0aGlzLl9hY3RpdmVCdWZmZXIueCtlLHRoaXMuX2FjdGl2ZUJ1ZmZlci55K3QpfSx0LnByb3RvdHlwZS5jdXJzb3JVcD1mdW5jdGlvbihlKXt2YXIgdD10aGlzLl9hY3RpdmVCdWZmZXIueS10aGlzLl9hY3RpdmVCdWZmZXIuc2Nyb2xsVG9wO3JldHVybiB0Pj0wP3RoaXMuX21vdmVDdXJzb3IoMCwtTWF0aC5taW4odCxlLnBhcmFtc1swXXx8MSkpOnRoaXMuX21vdmVDdXJzb3IoMCwtKGUucGFyYW1zWzBdfHwxKSksITB9LHQucHJvdG90eXBlLmN1cnNvckRvd249ZnVuY3Rpb24oZSl7dmFyIHQ9dGhpcy5fYWN0aXZlQnVmZmVyLnNjcm9sbEJvdHRvbS10aGlzLl9hY3RpdmVCdWZmZXIueTtyZXR1cm4gdD49MD90aGlzLl9tb3ZlQ3Vyc29yKDAsTWF0aC5taW4odCxlLnBhcmFtc1swXXx8MSkpOnRoaXMuX21vdmVDdXJzb3IoMCxlLnBhcmFtc1swXXx8MSksITB9LHQucHJvdG90eXBlLmN1cnNvckZvcndhcmQ9ZnVuY3Rpb24oZSl7cmV0dXJuIHRoaXMuX21vdmVDdXJzb3IoZS5wYXJhbXNbMF18fDEsMCksITB9LHQucHJvdG90eXBlLmN1cnNvckJhY2t3YXJkPWZ1bmN0aW9uKGUpe3JldHVybiB0aGlzLl9tb3ZlQ3Vyc29yKC0oZS5wYXJhbXNbMF18fDEpLDApLCEwfSx0LnByb3RvdHlwZS5jdXJzb3JOZXh0TGluZT1mdW5jdGlvbihlKXtyZXR1cm4gdGhpcy5jdXJzb3JEb3duKGUpLHRoaXMuX2FjdGl2ZUJ1ZmZlci54PTAsITB9LHQucHJvdG90eXBlLmN1cnNvclByZWNlZGluZ0xpbmU9ZnVuY3Rpb24oZSl7cmV0dXJuIHRoaXMuY3Vyc29yVXAoZSksdGhpcy5fYWN0aXZlQnVmZmVyLng9MCwhMH0sdC5wcm90b3R5cGUuY3Vyc29yQ2hhckFic29sdXRlPWZ1bmN0aW9uKGUpe3JldHVybiB0aGlzLl9zZXRDdXJzb3IoKGUucGFyYW1zWzBdfHwxKS0xLHRoaXMuX2FjdGl2ZUJ1ZmZlci55KSwhMH0sdC5wcm90b3R5cGUuY3Vyc29yUG9zaXRpb249ZnVuY3Rpb24oZSl7cmV0dXJuIHRoaXMuX3NldEN1cnNvcihlLmxlbmd0aD49Mj8oZS5wYXJhbXNbMV18fDEpLTE6MCwoZS5wYXJhbXNbMF18fDEpLTEpLCEwfSx0LnByb3RvdHlwZS5jaGFyUG9zQWJzb2x1dGU9ZnVuY3Rpb24oZSl7cmV0dXJuIHRoaXMuX3NldEN1cnNvcigoZS5wYXJhbXNbMF18fDEpLTEsdGhpcy5fYWN0aXZlQnVmZmVyLnkpLCEwfSx0LnByb3RvdHlwZS5oUG9zaXRpb25SZWxhdGl2ZT1mdW5jdGlvbihlKXtyZXR1cm4gdGhpcy5fbW92ZUN1cnNvcihlLnBhcmFtc1swXXx8MSwwKSwhMH0sdC5wcm90b3R5cGUubGluZVBvc0Fic29sdXRlPWZ1bmN0aW9uKGUpe3JldHVybiB0aGlzLl9zZXRDdXJzb3IodGhpcy5fYWN0aXZlQnVmZmVyLngsKGUucGFyYW1zWzBdfHwxKS0xKSwhMH0sdC5wcm90b3R5cGUudlBvc2l0aW9uUmVsYXRpdmU9ZnVuY3Rpb24oZSl7cmV0dXJuIHRoaXMuX21vdmVDdXJzb3IoMCxlLnBhcmFtc1swXXx8MSksITB9LHQucHJvdG90eXBlLmhWUG9zaXRpb249ZnVuY3Rpb24oZSl7cmV0dXJuIHRoaXMuY3Vyc29yUG9zaXRpb24oZSksITB9LHQucHJvdG90eXBlLnRhYkNsZWFyPWZ1bmN0aW9uKGUpe3ZhciB0PWUucGFyYW1zWzBdO3JldHVybiAwPT09dD9kZWxldGUgdGhpcy5fYWN0aXZlQnVmZmVyLnRhYnNbdGhpcy5fYWN0aXZlQnVmZmVyLnhdOjM9PT10JiYodGhpcy5fYWN0aXZlQnVmZmVyLnRhYnM9e30pLCEwfSx0LnByb3RvdHlwZS5jdXJzb3JGb3J3YXJkVGFiPWZ1bmN0aW9uKGUpe2lmKHRoaXMuX2FjdGl2ZUJ1ZmZlci54Pj10aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHMpcmV0dXJuITA7Zm9yKHZhciB0PWUucGFyYW1zWzBdfHwxO3QtLTspdGhpcy5fYWN0aXZlQnVmZmVyLng9dGhpcy5fYWN0aXZlQnVmZmVyLm5leHRTdG9wKCk7cmV0dXJuITB9LHQucHJvdG90eXBlLmN1cnNvckJhY2t3YXJkVGFiPWZ1bmN0aW9uKGUpe2lmKHRoaXMuX2FjdGl2ZUJ1ZmZlci54Pj10aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHMpcmV0dXJuITA7Zm9yKHZhciB0PWUucGFyYW1zWzBdfHwxO3QtLTspdGhpcy5fYWN0aXZlQnVmZmVyLng9dGhpcy5fYWN0aXZlQnVmZmVyLnByZXZTdG9wKCk7cmV0dXJuITB9LHQucHJvdG90eXBlLl9lcmFzZUluQnVmZmVyTGluZT1mdW5jdGlvbihlLHQscixpKXt2b2lkIDA9PT1pJiYoaT0hMSk7dmFyIG49dGhpcy5fYWN0aXZlQnVmZmVyLmxpbmVzLmdldCh0aGlzLl9hY3RpdmVCdWZmZXIueWJhc2UrZSk7bi5yZXBsYWNlQ2VsbHModCxyLHRoaXMuX2FjdGl2ZUJ1ZmZlci5nZXROdWxsQ2VsbCh0aGlzLl9lcmFzZUF0dHJEYXRhKCkpLHRoaXMuX2VyYXNlQXR0ckRhdGEoKSksaSYmKG4uaXNXcmFwcGVkPSExKX0sdC5wcm90b3R5cGUuX3Jlc2V0QnVmZmVyTGluZT1mdW5jdGlvbihlKXt2YXIgdD10aGlzLl9hY3RpdmVCdWZmZXIubGluZXMuZ2V0KHRoaXMuX2FjdGl2ZUJ1ZmZlci55YmFzZStlKTt0LmZpbGwodGhpcy5fYWN0aXZlQnVmZmVyLmdldE51bGxDZWxsKHRoaXMuX2VyYXNlQXR0ckRhdGEoKSkpLHQuaXNXcmFwcGVkPSExfSx0LnByb3RvdHlwZS5lcmFzZUluRGlzcGxheT1mdW5jdGlvbihlKXt2YXIgdDtzd2l0Y2godGhpcy5fcmVzdHJpY3RDdXJzb3IodGhpcy5fYnVmZmVyU2VydmljZS5jb2xzKSxlLnBhcmFtc1swXSl7Y2FzZSAwOmZvcih0PXRoaXMuX2FjdGl2ZUJ1ZmZlci55LHRoaXMuX2RpcnR5Um93U2VydmljZS5tYXJrRGlydHkodCksdGhpcy5fZXJhc2VJbkJ1ZmZlckxpbmUodCsrLHRoaXMuX2FjdGl2ZUJ1ZmZlci54LHRoaXMuX2J1ZmZlclNlcnZpY2UuY29scywwPT09dGhpcy5fYWN0aXZlQnVmZmVyLngpO3Q8dGhpcy5fYnVmZmVyU2VydmljZS5yb3dzO3QrKyl0aGlzLl9yZXNldEJ1ZmZlckxpbmUodCk7dGhpcy5fZGlydHlSb3dTZXJ2aWNlLm1hcmtEaXJ0eSh0KTticmVhaztjYXNlIDE6Zm9yKHQ9dGhpcy5fYWN0aXZlQnVmZmVyLnksdGhpcy5fZGlydHlSb3dTZXJ2aWNlLm1hcmtEaXJ0eSh0KSx0aGlzLl9lcmFzZUluQnVmZmVyTGluZSh0LDAsdGhpcy5fYWN0aXZlQnVmZmVyLngrMSwhMCksdGhpcy5fYWN0aXZlQnVmZmVyLngrMT49dGhpcy5fYnVmZmVyU2VydmljZS5jb2xzJiYodGhpcy5fYWN0aXZlQnVmZmVyLmxpbmVzLmdldCh0KzEpLmlzV3JhcHBlZD0hMSk7dC0tOyl0aGlzLl9yZXNldEJ1ZmZlckxpbmUodCk7dGhpcy5fZGlydHlSb3dTZXJ2aWNlLm1hcmtEaXJ0eSgwKTticmVhaztjYXNlIDI6Zm9yKHQ9dGhpcy5fYnVmZmVyU2VydmljZS5yb3dzLHRoaXMuX2RpcnR5Um93U2VydmljZS5tYXJrRGlydHkodC0xKTt0LS07KXRoaXMuX3Jlc2V0QnVmZmVyTGluZSh0KTt0aGlzLl9kaXJ0eVJvd1NlcnZpY2UubWFya0RpcnR5KDApO2JyZWFrO2Nhc2UgMzp2YXIgcj10aGlzLl9hY3RpdmVCdWZmZXIubGluZXMubGVuZ3RoLXRoaXMuX2J1ZmZlclNlcnZpY2Uucm93cztyPjAmJih0aGlzLl9hY3RpdmVCdWZmZXIubGluZXMudHJpbVN0YXJ0KHIpLHRoaXMuX2FjdGl2ZUJ1ZmZlci55YmFzZT1NYXRoLm1heCh0aGlzLl9hY3RpdmVCdWZmZXIueWJhc2UtciwwKSx0aGlzLl9hY3RpdmVCdWZmZXIueWRpc3A9TWF0aC5tYXgodGhpcy5fYWN0aXZlQnVmZmVyLnlkaXNwLXIsMCksdGhpcy5fb25TY3JvbGwuZmlyZSgwKSl9cmV0dXJuITB9LHQucHJvdG90eXBlLmVyYXNlSW5MaW5lPWZ1bmN0aW9uKGUpe3N3aXRjaCh0aGlzLl9yZXN0cmljdEN1cnNvcih0aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHMpLGUucGFyYW1zWzBdKXtjYXNlIDA6dGhpcy5fZXJhc2VJbkJ1ZmZlckxpbmUodGhpcy5fYWN0aXZlQnVmZmVyLnksdGhpcy5fYWN0aXZlQnVmZmVyLngsdGhpcy5fYnVmZmVyU2VydmljZS5jb2xzLDA9PT10aGlzLl9hY3RpdmVCdWZmZXIueCk7YnJlYWs7Y2FzZSAxOnRoaXMuX2VyYXNlSW5CdWZmZXJMaW5lKHRoaXMuX2FjdGl2ZUJ1ZmZlci55LDAsdGhpcy5fYWN0aXZlQnVmZmVyLngrMSwhMSk7YnJlYWs7Y2FzZSAyOnRoaXMuX2VyYXNlSW5CdWZmZXJMaW5lKHRoaXMuX2FjdGl2ZUJ1ZmZlci55LDAsdGhpcy5fYnVmZmVyU2VydmljZS5jb2xzLCEwKX1yZXR1cm4gdGhpcy5fZGlydHlSb3dTZXJ2aWNlLm1hcmtEaXJ0eSh0aGlzLl9hY3RpdmVCdWZmZXIueSksITB9LHQucHJvdG90eXBlLmluc2VydExpbmVzPWZ1bmN0aW9uKGUpe3RoaXMuX3Jlc3RyaWN0Q3Vyc29yKCk7dmFyIHQ9ZS5wYXJhbXNbMF18fDE7aWYodGhpcy5fYWN0aXZlQnVmZmVyLnk+dGhpcy5fYWN0aXZlQnVmZmVyLnNjcm9sbEJvdHRvbXx8dGhpcy5fYWN0aXZlQnVmZmVyLnk8dGhpcy5fYWN0aXZlQnVmZmVyLnNjcm9sbFRvcClyZXR1cm4hMDtmb3IodmFyIHI9dGhpcy5fYWN0aXZlQnVmZmVyLnliYXNlK3RoaXMuX2FjdGl2ZUJ1ZmZlci55LGk9dGhpcy5fYnVmZmVyU2VydmljZS5yb3dzLTEtdGhpcy5fYWN0aXZlQnVmZmVyLnNjcm9sbEJvdHRvbSxuPXRoaXMuX2J1ZmZlclNlcnZpY2Uucm93cy0xK3RoaXMuX2FjdGl2ZUJ1ZmZlci55YmFzZS1pKzE7dC0tOyl0aGlzLl9hY3RpdmVCdWZmZXIubGluZXMuc3BsaWNlKG4tMSwxKSx0aGlzLl9hY3RpdmVCdWZmZXIubGluZXMuc3BsaWNlKHIsMCx0aGlzLl9hY3RpdmVCdWZmZXIuZ2V0QmxhbmtMaW5lKHRoaXMuX2VyYXNlQXR0ckRhdGEoKSkpO3JldHVybiB0aGlzLl9kaXJ0eVJvd1NlcnZpY2UubWFya1JhbmdlRGlydHkodGhpcy5fYWN0aXZlQnVmZmVyLnksdGhpcy5fYWN0aXZlQnVmZmVyLnNjcm9sbEJvdHRvbSksdGhpcy5fYWN0aXZlQnVmZmVyLng9MCwhMH0sdC5wcm90b3R5cGUuZGVsZXRlTGluZXM9ZnVuY3Rpb24oZSl7dGhpcy5fcmVzdHJpY3RDdXJzb3IoKTt2YXIgdD1lLnBhcmFtc1swXXx8MTtpZih0aGlzLl9hY3RpdmVCdWZmZXIueT50aGlzLl9hY3RpdmVCdWZmZXIuc2Nyb2xsQm90dG9tfHx0aGlzLl9hY3RpdmVCdWZmZXIueTx0aGlzLl9hY3RpdmVCdWZmZXIuc2Nyb2xsVG9wKXJldHVybiEwO3ZhciByLGk9dGhpcy5fYWN0aXZlQnVmZmVyLnliYXNlK3RoaXMuX2FjdGl2ZUJ1ZmZlci55O2ZvcihyPXRoaXMuX2J1ZmZlclNlcnZpY2Uucm93cy0xLXRoaXMuX2FjdGl2ZUJ1ZmZlci5zY3JvbGxCb3R0b20scj10aGlzLl9idWZmZXJTZXJ2aWNlLnJvd3MtMSt0aGlzLl9hY3RpdmVCdWZmZXIueWJhc2Utcjt0LS07KXRoaXMuX2FjdGl2ZUJ1ZmZlci5saW5lcy5zcGxpY2UoaSwxKSx0aGlzLl9hY3RpdmVCdWZmZXIubGluZXMuc3BsaWNlKHIsMCx0aGlzLl9hY3RpdmVCdWZmZXIuZ2V0QmxhbmtMaW5lKHRoaXMuX2VyYXNlQXR0ckRhdGEoKSkpO3JldHVybiB0aGlzLl9kaXJ0eVJvd1NlcnZpY2UubWFya1JhbmdlRGlydHkodGhpcy5fYWN0aXZlQnVmZmVyLnksdGhpcy5fYWN0aXZlQnVmZmVyLnNjcm9sbEJvdHRvbSksdGhpcy5fYWN0aXZlQnVmZmVyLng9MCwhMH0sdC5wcm90b3R5cGUuaW5zZXJ0Q2hhcnM9ZnVuY3Rpb24oZSl7dGhpcy5fcmVzdHJpY3RDdXJzb3IoKTt2YXIgdD10aGlzLl9hY3RpdmVCdWZmZXIubGluZXMuZ2V0KHRoaXMuX2FjdGl2ZUJ1ZmZlci55YmFzZSt0aGlzLl9hY3RpdmVCdWZmZXIueSk7cmV0dXJuIHQmJih0Lmluc2VydENlbGxzKHRoaXMuX2FjdGl2ZUJ1ZmZlci54LGUucGFyYW1zWzBdfHwxLHRoaXMuX2FjdGl2ZUJ1ZmZlci5nZXROdWxsQ2VsbCh0aGlzLl9lcmFzZUF0dHJEYXRhKCkpLHRoaXMuX2VyYXNlQXR0ckRhdGEoKSksdGhpcy5fZGlydHlSb3dTZXJ2aWNlLm1hcmtEaXJ0eSh0aGlzLl9hY3RpdmVCdWZmZXIueSkpLCEwfSx0LnByb3RvdHlwZS5kZWxldGVDaGFycz1mdW5jdGlvbihlKXt0aGlzLl9yZXN0cmljdEN1cnNvcigpO3ZhciB0PXRoaXMuX2FjdGl2ZUJ1ZmZlci5saW5lcy5nZXQodGhpcy5fYWN0aXZlQnVmZmVyLnliYXNlK3RoaXMuX2FjdGl2ZUJ1ZmZlci55KTtyZXR1cm4gdCYmKHQuZGVsZXRlQ2VsbHModGhpcy5fYWN0aXZlQnVmZmVyLngsZS5wYXJhbXNbMF18fDEsdGhpcy5fYWN0aXZlQnVmZmVyLmdldE51bGxDZWxsKHRoaXMuX2VyYXNlQXR0ckRhdGEoKSksdGhpcy5fZXJhc2VBdHRyRGF0YSgpKSx0aGlzLl9kaXJ0eVJvd1NlcnZpY2UubWFya0RpcnR5KHRoaXMuX2FjdGl2ZUJ1ZmZlci55KSksITB9LHQucHJvdG90eXBlLnNjcm9sbFVwPWZ1bmN0aW9uKGUpe2Zvcih2YXIgdD1lLnBhcmFtc1swXXx8MTt0LS07KXRoaXMuX2FjdGl2ZUJ1ZmZlci5saW5lcy5zcGxpY2UodGhpcy5fYWN0aXZlQnVmZmVyLnliYXNlK3RoaXMuX2FjdGl2ZUJ1ZmZlci5zY3JvbGxUb3AsMSksdGhpcy5fYWN0aXZlQnVmZmVyLmxpbmVzLnNwbGljZSh0aGlzLl9hY3RpdmVCdWZmZXIueWJhc2UrdGhpcy5fYWN0aXZlQnVmZmVyLnNjcm9sbEJvdHRvbSwwLHRoaXMuX2FjdGl2ZUJ1ZmZlci5nZXRCbGFua0xpbmUodGhpcy5fZXJhc2VBdHRyRGF0YSgpKSk7cmV0dXJuIHRoaXMuX2RpcnR5Um93U2VydmljZS5tYXJrUmFuZ2VEaXJ0eSh0aGlzLl9hY3RpdmVCdWZmZXIuc2Nyb2xsVG9wLHRoaXMuX2FjdGl2ZUJ1ZmZlci5zY3JvbGxCb3R0b20pLCEwfSx0LnByb3RvdHlwZS5zY3JvbGxEb3duPWZ1bmN0aW9uKGUpe2Zvcih2YXIgdD1lLnBhcmFtc1swXXx8MTt0LS07KXRoaXMuX2FjdGl2ZUJ1ZmZlci5saW5lcy5zcGxpY2UodGhpcy5fYWN0aXZlQnVmZmVyLnliYXNlK3RoaXMuX2FjdGl2ZUJ1ZmZlci5zY3JvbGxCb3R0b20sMSksdGhpcy5fYWN0aXZlQnVmZmVyLmxpbmVzLnNwbGljZSh0aGlzLl9hY3RpdmVCdWZmZXIueWJhc2UrdGhpcy5fYWN0aXZlQnVmZmVyLnNjcm9sbFRvcCwwLHRoaXMuX2FjdGl2ZUJ1ZmZlci5nZXRCbGFua0xpbmUoZi5ERUZBVUxUX0FUVFJfREFUQSkpO3JldHVybiB0aGlzLl9kaXJ0eVJvd1NlcnZpY2UubWFya1JhbmdlRGlydHkodGhpcy5fYWN0aXZlQnVmZmVyLnNjcm9sbFRvcCx0aGlzLl9hY3RpdmVCdWZmZXIuc2Nyb2xsQm90dG9tKSwhMH0sdC5wcm90b3R5cGUuc2Nyb2xsTGVmdD1mdW5jdGlvbihlKXtpZih0aGlzLl9hY3RpdmVCdWZmZXIueT50aGlzLl9hY3RpdmVCdWZmZXIuc2Nyb2xsQm90dG9tfHx0aGlzLl9hY3RpdmVCdWZmZXIueTx0aGlzLl9hY3RpdmVCdWZmZXIuc2Nyb2xsVG9wKXJldHVybiEwO2Zvcih2YXIgdD1lLnBhcmFtc1swXXx8MSxyPXRoaXMuX2FjdGl2ZUJ1ZmZlci5zY3JvbGxUb3A7cjw9dGhpcy5fYWN0aXZlQnVmZmVyLnNjcm9sbEJvdHRvbTsrK3Ipe3ZhciBpPXRoaXMuX2FjdGl2ZUJ1ZmZlci5saW5lcy5nZXQodGhpcy5fYWN0aXZlQnVmZmVyLnliYXNlK3IpO2kuZGVsZXRlQ2VsbHMoMCx0LHRoaXMuX2FjdGl2ZUJ1ZmZlci5nZXROdWxsQ2VsbCh0aGlzLl9lcmFzZUF0dHJEYXRhKCkpLHRoaXMuX2VyYXNlQXR0ckRhdGEoKSksaS5pc1dyYXBwZWQ9ITF9cmV0dXJuIHRoaXMuX2RpcnR5Um93U2VydmljZS5tYXJrUmFuZ2VEaXJ0eSh0aGlzLl9hY3RpdmVCdWZmZXIuc2Nyb2xsVG9wLHRoaXMuX2FjdGl2ZUJ1ZmZlci5zY3JvbGxCb3R0b20pLCEwfSx0LnByb3RvdHlwZS5zY3JvbGxSaWdodD1mdW5jdGlvbihlKXtpZih0aGlzLl9hY3RpdmVCdWZmZXIueT50aGlzLl9hY3RpdmVCdWZmZXIuc2Nyb2xsQm90dG9tfHx0aGlzLl9hY3RpdmVCdWZmZXIueTx0aGlzLl9hY3RpdmVCdWZmZXIuc2Nyb2xsVG9wKXJldHVybiEwO2Zvcih2YXIgdD1lLnBhcmFtc1swXXx8MSxyPXRoaXMuX2FjdGl2ZUJ1ZmZlci5zY3JvbGxUb3A7cjw9dGhpcy5fYWN0aXZlQnVmZmVyLnNjcm9sbEJvdHRvbTsrK3Ipe3ZhciBpPXRoaXMuX2FjdGl2ZUJ1ZmZlci5saW5lcy5nZXQodGhpcy5fYWN0aXZlQnVmZmVyLnliYXNlK3IpO2kuaW5zZXJ0Q2VsbHMoMCx0LHRoaXMuX2FjdGl2ZUJ1ZmZlci5nZXROdWxsQ2VsbCh0aGlzLl9lcmFzZUF0dHJEYXRhKCkpLHRoaXMuX2VyYXNlQXR0ckRhdGEoKSksaS5pc1dyYXBwZWQ9ITF9cmV0dXJuIHRoaXMuX2RpcnR5Um93U2VydmljZS5tYXJrUmFuZ2VEaXJ0eSh0aGlzLl9hY3RpdmVCdWZmZXIuc2Nyb2xsVG9wLHRoaXMuX2FjdGl2ZUJ1ZmZlci5zY3JvbGxCb3R0b20pLCEwfSx0LnByb3RvdHlwZS5pbnNlcnRDb2x1bW5zPWZ1bmN0aW9uKGUpe2lmKHRoaXMuX2FjdGl2ZUJ1ZmZlci55PnRoaXMuX2FjdGl2ZUJ1ZmZlci5zY3JvbGxCb3R0b218fHRoaXMuX2FjdGl2ZUJ1ZmZlci55PHRoaXMuX2FjdGl2ZUJ1ZmZlci5zY3JvbGxUb3ApcmV0dXJuITA7Zm9yKHZhciB0PWUucGFyYW1zWzBdfHwxLHI9dGhpcy5fYWN0aXZlQnVmZmVyLnNjcm9sbFRvcDtyPD10aGlzLl9hY3RpdmVCdWZmZXIuc2Nyb2xsQm90dG9tOysrcil7dmFyIGk9dGhpcy5fYWN0aXZlQnVmZmVyLmxpbmVzLmdldCh0aGlzLl9hY3RpdmVCdWZmZXIueWJhc2Urcik7aS5pbnNlcnRDZWxscyh0aGlzLl9hY3RpdmVCdWZmZXIueCx0LHRoaXMuX2FjdGl2ZUJ1ZmZlci5nZXROdWxsQ2VsbCh0aGlzLl9lcmFzZUF0dHJEYXRhKCkpLHRoaXMuX2VyYXNlQXR0ckRhdGEoKSksaS5pc1dyYXBwZWQ9ITF9cmV0dXJuIHRoaXMuX2RpcnR5Um93U2VydmljZS5tYXJrUmFuZ2VEaXJ0eSh0aGlzLl9hY3RpdmVCdWZmZXIuc2Nyb2xsVG9wLHRoaXMuX2FjdGl2ZUJ1ZmZlci5zY3JvbGxCb3R0b20pLCEwfSx0LnByb3RvdHlwZS5kZWxldGVDb2x1bW5zPWZ1bmN0aW9uKGUpe2lmKHRoaXMuX2FjdGl2ZUJ1ZmZlci55PnRoaXMuX2FjdGl2ZUJ1ZmZlci5zY3JvbGxCb3R0b218fHRoaXMuX2FjdGl2ZUJ1ZmZlci55PHRoaXMuX2FjdGl2ZUJ1ZmZlci5zY3JvbGxUb3ApcmV0dXJuITA7Zm9yKHZhciB0PWUucGFyYW1zWzBdfHwxLHI9dGhpcy5fYWN0aXZlQnVmZmVyLnNjcm9sbFRvcDtyPD10aGlzLl9hY3RpdmVCdWZmZXIuc2Nyb2xsQm90dG9tOysrcil7dmFyIGk9dGhpcy5fYWN0aXZlQnVmZmVyLmxpbmVzLmdldCh0aGlzLl9hY3RpdmVCdWZmZXIueWJhc2Urcik7aS5kZWxldGVDZWxscyh0aGlzLl9hY3RpdmVCdWZmZXIueCx0LHRoaXMuX2FjdGl2ZUJ1ZmZlci5nZXROdWxsQ2VsbCh0aGlzLl9lcmFzZUF0dHJEYXRhKCkpLHRoaXMuX2VyYXNlQXR0ckRhdGEoKSksaS5pc1dyYXBwZWQ9ITF9cmV0dXJuIHRoaXMuX2RpcnR5Um93U2VydmljZS5tYXJrUmFuZ2VEaXJ0eSh0aGlzLl9hY3RpdmVCdWZmZXIuc2Nyb2xsVG9wLHRoaXMuX2FjdGl2ZUJ1ZmZlci5zY3JvbGxCb3R0b20pLCEwfSx0LnByb3RvdHlwZS5lcmFzZUNoYXJzPWZ1bmN0aW9uKGUpe3RoaXMuX3Jlc3RyaWN0Q3Vyc29yKCk7dmFyIHQ9dGhpcy5fYWN0aXZlQnVmZmVyLmxpbmVzLmdldCh0aGlzLl9hY3RpdmVCdWZmZXIueWJhc2UrdGhpcy5fYWN0aXZlQnVmZmVyLnkpO3JldHVybiB0JiYodC5yZXBsYWNlQ2VsbHModGhpcy5fYWN0aXZlQnVmZmVyLngsdGhpcy5fYWN0aXZlQnVmZmVyLngrKGUucGFyYW1zWzBdfHwxKSx0aGlzLl9hY3RpdmVCdWZmZXIuZ2V0TnVsbENlbGwodGhpcy5fZXJhc2VBdHRyRGF0YSgpKSx0aGlzLl9lcmFzZUF0dHJEYXRhKCkpLHRoaXMuX2RpcnR5Um93U2VydmljZS5tYXJrRGlydHkodGhpcy5fYWN0aXZlQnVmZmVyLnkpKSwhMH0sdC5wcm90b3R5cGUucmVwZWF0UHJlY2VkaW5nQ2hhcmFjdGVyPWZ1bmN0aW9uKGUpe2lmKCF0aGlzLl9wYXJzZXIucHJlY2VkaW5nQ29kZXBvaW50KXJldHVybiEwO2Zvcih2YXIgdD1lLnBhcmFtc1swXXx8MSxyPW5ldyBVaW50MzJBcnJheSh0KSxpPTA7aTx0OysraSlyW2ldPXRoaXMuX3BhcnNlci5wcmVjZWRpbmdDb2RlcG9pbnQ7cmV0dXJuIHRoaXMucHJpbnQociwwLHIubGVuZ3RoKSwhMH0sdC5wcm90b3R5cGUuc2VuZERldmljZUF0dHJpYnV0ZXNQcmltYXJ5PWZ1bmN0aW9uKGUpe3JldHVybiBlLnBhcmFtc1swXT4wfHwodGhpcy5faXMoInh0ZXJtIil8fHRoaXMuX2lzKCJyeHZ0LXVuaWNvZGUiKXx8dGhpcy5faXMoInNjcmVlbiIpP3RoaXMuX2NvcmVTZXJ2aWNlLnRyaWdnZXJEYXRhRXZlbnQocy5DMC5FU0MrIls/MTsyYyIpOnRoaXMuX2lzKCJsaW51eCIpJiZ0aGlzLl9jb3JlU2VydmljZS50cmlnZ2VyRGF0YUV2ZW50KHMuQzAuRVNDKyJbPzZjIikpLCEwfSx0LnByb3RvdHlwZS5zZW5kRGV2aWNlQXR0cmlidXRlc1NlY29uZGFyeT1mdW5jdGlvbihlKXtyZXR1cm4gZS5wYXJhbXNbMF0+MHx8KHRoaXMuX2lzKCJ4dGVybSIpP3RoaXMuX2NvcmVTZXJ2aWNlLnRyaWdnZXJEYXRhRXZlbnQocy5DMC5FU0MrIls+MDsyNzY7MGMiKTp0aGlzLl9pcygicnh2dC11bmljb2RlIik/dGhpcy5fY29yZVNlcnZpY2UudHJpZ2dlckRhdGFFdmVudChzLkMwLkVTQysiWz44NTs5NTswYyIpOnRoaXMuX2lzKCJsaW51eCIpP3RoaXMuX2NvcmVTZXJ2aWNlLnRyaWdnZXJEYXRhRXZlbnQoZS5wYXJhbXNbMF0rImMiKTp0aGlzLl9pcygic2NyZWVuIikmJnRoaXMuX2NvcmVTZXJ2aWNlLnRyaWdnZXJEYXRhRXZlbnQocy5DMC5FU0MrIls+ODM7NDAwMDM7MGMiKSksITB9LHQucHJvdG90eXBlLl9pcz1mdW5jdGlvbihlKXtyZXR1cm4gMD09PSh0aGlzLl9vcHRpb25zU2VydmljZS5vcHRpb25zLnRlcm1OYW1lKyIiKS5pbmRleE9mKGUpfSx0LnByb3RvdHlwZS5zZXRNb2RlPWZ1bmN0aW9uKGUpe2Zvcih2YXIgdD0wO3Q8ZS5sZW5ndGg7dCsrKTQ9PT1lLnBhcmFtc1t0XSYmKHRoaXMuX2NvcmVTZXJ2aWNlLm1vZGVzLmluc2VydE1vZGU9ITApO3JldHVybiEwfSx0LnByb3RvdHlwZS5zZXRNb2RlUHJpdmF0ZT1mdW5jdGlvbihlKXtmb3IodmFyIHQ9MDt0PGUubGVuZ3RoO3QrKylzd2l0Y2goZS5wYXJhbXNbdF0pe2Nhc2UgMTp0aGlzLl9jb3JlU2VydmljZS5kZWNQcml2YXRlTW9kZXMuYXBwbGljYXRpb25DdXJzb3JLZXlzPSEwO2JyZWFrO2Nhc2UgMjp0aGlzLl9jaGFyc2V0U2VydmljZS5zZXRnQ2hhcnNldCgwLGEuREVGQVVMVF9DSEFSU0VUKSx0aGlzLl9jaGFyc2V0U2VydmljZS5zZXRnQ2hhcnNldCgxLGEuREVGQVVMVF9DSEFSU0VUKSx0aGlzLl9jaGFyc2V0U2VydmljZS5zZXRnQ2hhcnNldCgyLGEuREVGQVVMVF9DSEFSU0VUKSx0aGlzLl9jaGFyc2V0U2VydmljZS5zZXRnQ2hhcnNldCgzLGEuREVGQVVMVF9DSEFSU0VUKTticmVhaztjYXNlIDM6dGhpcy5fb3B0aW9uc1NlcnZpY2Uub3B0aW9ucy53aW5kb3dPcHRpb25zLnNldFdpbkxpbmVzJiYodGhpcy5fYnVmZmVyU2VydmljZS5yZXNpemUoMTMyLHRoaXMuX2J1ZmZlclNlcnZpY2Uucm93cyksdGhpcy5fb25SZXF1ZXN0UmVzZXQuZmlyZSgpKTticmVhaztjYXNlIDY6dGhpcy5fY29yZVNlcnZpY2UuZGVjUHJpdmF0ZU1vZGVzLm9yaWdpbj0hMCx0aGlzLl9zZXRDdXJzb3IoMCwwKTticmVhaztjYXNlIDc6dGhpcy5fY29yZVNlcnZpY2UuZGVjUHJpdmF0ZU1vZGVzLndyYXBhcm91bmQ9ITA7YnJlYWs7Y2FzZSAxMjpicmVhaztjYXNlIDQ1OnRoaXMuX2NvcmVTZXJ2aWNlLmRlY1ByaXZhdGVNb2Rlcy5yZXZlcnNlV3JhcGFyb3VuZD0hMDticmVhaztjYXNlIDY2OnRoaXMuX2xvZ1NlcnZpY2UuZGVidWcoIlNlcmlhbCBwb3J0IHJlcXVlc3RlZCBhcHBsaWNhdGlvbiBrZXlwYWQuIiksdGhpcy5fY29yZVNlcnZpY2UuZGVjUHJpdmF0ZU1vZGVzLmFwcGxpY2F0aW9uS2V5cGFkPSEwLHRoaXMuX29uUmVxdWVzdFN5bmNTY3JvbGxCYXIuZmlyZSgpO2JyZWFrO2Nhc2UgOTp0aGlzLl9jb3JlTW91c2VTZXJ2aWNlLmFjdGl2ZVByb3RvY29sPSJYMTAiO2JyZWFrO2Nhc2UgMWUzOnRoaXMuX2NvcmVNb3VzZVNlcnZpY2UuYWN0aXZlUHJvdG9jb2w9IlZUMjAwIjticmVhaztjYXNlIDEwMDI6dGhpcy5fY29yZU1vdXNlU2VydmljZS5hY3RpdmVQcm90b2NvbD0iRFJBRyI7YnJlYWs7Y2FzZSAxMDAzOnRoaXMuX2NvcmVNb3VzZVNlcnZpY2UuYWN0aXZlUHJvdG9jb2w9IkFOWSI7YnJlYWs7Y2FzZSAxMDA0OnRoaXMuX2NvcmVTZXJ2aWNlLmRlY1ByaXZhdGVNb2Rlcy5zZW5kRm9jdXM9ITAsdGhpcy5fb25SZXF1ZXN0U2VuZEZvY3VzLmZpcmUoKTticmVhaztjYXNlIDEwMDU6dGhpcy5fbG9nU2VydmljZS5kZWJ1ZygiREVDU0VUIDEwMDUgbm90IHN1cHBvcnRlZCAoc2VlICMyNTA3KSIpO2JyZWFrO2Nhc2UgMTAwNjp0aGlzLl9jb3JlTW91c2VTZXJ2aWNlLmFjdGl2ZUVuY29kaW5nPSJTR1IiO2JyZWFrO2Nhc2UgMTAxNTp0aGlzLl9sb2dTZXJ2aWNlLmRlYnVnKCJERUNTRVQgMTAxNSBub3Qgc3VwcG9ydGVkIChzZWUgIzI1MDcpIik7YnJlYWs7Y2FzZSAyNTp0aGlzLl9jb3JlU2VydmljZS5pc0N1cnNvckhpZGRlbj0hMTticmVhaztjYXNlIDEwNDg6dGhpcy5zYXZlQ3Vyc29yKCk7YnJlYWs7Y2FzZSAxMDQ5OnRoaXMuc2F2ZUN1cnNvcigpO2Nhc2UgNDc6Y2FzZSAxMDQ3OnRoaXMuX2J1ZmZlclNlcnZpY2UuYnVmZmVycy5hY3RpdmF0ZUFsdEJ1ZmZlcih0aGlzLl9lcmFzZUF0dHJEYXRhKCkpLHRoaXMuX2NvcmVTZXJ2aWNlLmlzQ3Vyc29ySW5pdGlhbGl6ZWQ9ITAsdGhpcy5fb25SZXF1ZXN0UmVmcmVzaFJvd3MuZmlyZSgwLHRoaXMuX2J1ZmZlclNlcnZpY2Uucm93cy0xKSx0aGlzLl9vblJlcXVlc3RTeW5jU2Nyb2xsQmFyLmZpcmUoKTticmVhaztjYXNlIDIwMDQ6dGhpcy5fY29yZVNlcnZpY2UuZGVjUHJpdmF0ZU1vZGVzLmJyYWNrZXRlZFBhc3RlTW9kZT0hMH1yZXR1cm4hMH0sdC5wcm90b3R5cGUucmVzZXRNb2RlPWZ1bmN0aW9uKGUpe2Zvcih2YXIgdD0wO3Q8ZS5sZW5ndGg7dCsrKTQ9PT1lLnBhcmFtc1t0XSYmKHRoaXMuX2NvcmVTZXJ2aWNlLm1vZGVzLmluc2VydE1vZGU9ITEpO3JldHVybiEwfSx0LnByb3RvdHlwZS5yZXNldE1vZGVQcml2YXRlPWZ1bmN0aW9uKGUpe2Zvcih2YXIgdD0wO3Q8ZS5sZW5ndGg7dCsrKXN3aXRjaChlLnBhcmFtc1t0XSl7Y2FzZSAxOnRoaXMuX2NvcmVTZXJ2aWNlLmRlY1ByaXZhdGVNb2Rlcy5hcHBsaWNhdGlvbkN1cnNvcktleXM9ITE7YnJlYWs7Y2FzZSAzOnRoaXMuX29wdGlvbnNTZXJ2aWNlLm9wdGlvbnMud2luZG93T3B0aW9ucy5zZXRXaW5MaW5lcyYmKHRoaXMuX2J1ZmZlclNlcnZpY2UucmVzaXplKDgwLHRoaXMuX2J1ZmZlclNlcnZpY2Uucm93cyksdGhpcy5fb25SZXF1ZXN0UmVzZXQuZmlyZSgpKTticmVhaztjYXNlIDY6dGhpcy5fY29yZVNlcnZpY2UuZGVjUHJpdmF0ZU1vZGVzLm9yaWdpbj0hMSx0aGlzLl9zZXRDdXJzb3IoMCwwKTticmVhaztjYXNlIDc6dGhpcy5fY29yZVNlcnZpY2UuZGVjUHJpdmF0ZU1vZGVzLndyYXBhcm91bmQ9ITE7YnJlYWs7Y2FzZSAxMjpicmVhaztjYXNlIDQ1OnRoaXMuX2NvcmVTZXJ2aWNlLmRlY1ByaXZhdGVNb2Rlcy5yZXZlcnNlV3JhcGFyb3VuZD0hMTticmVhaztjYXNlIDY2OnRoaXMuX2xvZ1NlcnZpY2UuZGVidWcoIlN3aXRjaGluZyBiYWNrIHRvIG5vcm1hbCBrZXlwYWQuIiksdGhpcy5fY29yZVNlcnZpY2UuZGVjUHJpdmF0ZU1vZGVzLmFwcGxpY2F0aW9uS2V5cGFkPSExLHRoaXMuX29uUmVxdWVzdFN5bmNTY3JvbGxCYXIuZmlyZSgpO2JyZWFrO2Nhc2UgOTpjYXNlIDFlMzpjYXNlIDEwMDI6Y2FzZSAxMDAzOnRoaXMuX2NvcmVNb3VzZVNlcnZpY2UuYWN0aXZlUHJvdG9jb2w9Ik5PTkUiO2JyZWFrO2Nhc2UgMTAwNDp0aGlzLl9jb3JlU2VydmljZS5kZWNQcml2YXRlTW9kZXMuc2VuZEZvY3VzPSExO2JyZWFrO2Nhc2UgMTAwNTp0aGlzLl9sb2dTZXJ2aWNlLmRlYnVnKCJERUNSU1QgMTAwNSBub3Qgc3VwcG9ydGVkIChzZWUgIzI1MDcpIik7YnJlYWs7Y2FzZSAxMDA2OnRoaXMuX2NvcmVNb3VzZVNlcnZpY2UuYWN0aXZlRW5jb2Rpbmc9IkRFRkFVTFQiO2JyZWFrO2Nhc2UgMTAxNTp0aGlzLl9sb2dTZXJ2aWNlLmRlYnVnKCJERUNSU1QgMTAxNSBub3Qgc3VwcG9ydGVkIChzZWUgIzI1MDcpIik7YnJlYWs7Y2FzZSAyNTp0aGlzLl9jb3JlU2VydmljZS5pc0N1cnNvckhpZGRlbj0hMDticmVhaztjYXNlIDEwNDg6dGhpcy5yZXN0b3JlQ3Vyc29yKCk7YnJlYWs7Y2FzZSAxMDQ5OmNhc2UgNDc6Y2FzZSAxMDQ3OnRoaXMuX2J1ZmZlclNlcnZpY2UuYnVmZmVycy5hY3RpdmF0ZU5vcm1hbEJ1ZmZlcigpLDEwNDk9PT1lLnBhcmFtc1t0XSYmdGhpcy5yZXN0b3JlQ3Vyc29yKCksdGhpcy5fY29yZVNlcnZpY2UuaXNDdXJzb3JJbml0aWFsaXplZD0hMCx0aGlzLl9vblJlcXVlc3RSZWZyZXNoUm93cy5maXJlKDAsdGhpcy5fYnVmZmVyU2VydmljZS5yb3dzLTEpLHRoaXMuX29uUmVxdWVzdFN5bmNTY3JvbGxCYXIuZmlyZSgpO2JyZWFrO2Nhc2UgMjAwNDp0aGlzLl9jb3JlU2VydmljZS5kZWNQcml2YXRlTW9kZXMuYnJhY2tldGVkUGFzdGVNb2RlPSExfXJldHVybiEwfSx0LnByb3RvdHlwZS5fdXBkYXRlQXR0ckNvbG9yPWZ1bmN0aW9uKGUsdCxyLGksbil7cmV0dXJuIDI9PT10PyhlfD01MDMzMTY0OCxlJj0tMTY3NzcyMTYsZXw9di5BdHRyaWJ1dGVEYXRhLmZyb21Db2xvclJHQihbcixpLG5dKSk6NT09PXQmJihlJj0tNTAzMzE5MDQsZXw9MzM1NTQ0MzJ8MjU1JnIpLGV9LHQucHJvdG90eXBlLl9leHRyYWN0Q29sb3I9ZnVuY3Rpb24oZSx0LHIpe3ZhciBpPVswLDAsLTEsMCwwLDBdLG49MCxvPTA7ZG97aWYoaVtvK25dPWUucGFyYW1zW3Qrb10sZS5oYXNTdWJQYXJhbXModCtvKSl7dmFyIHM9ZS5nZXRTdWJQYXJhbXModCtvKSxhPTA7ZG97NT09PWlbMV0mJihuPTEpLGlbbythKzErbl09c1thXX13aGlsZSgrK2E8cy5sZW5ndGgmJmErbysxK248aS5sZW5ndGgpO2JyZWFrfWlmKDU9PT1pWzFdJiZvK24+PTJ8fDI9PT1pWzFdJiZvK24+PTUpYnJlYWs7aVsxXSYmKG49MSl9d2hpbGUoKytvK3Q8ZS5sZW5ndGgmJm8rbjxpLmxlbmd0aCk7Zm9yKGE9MjthPGkubGVuZ3RoOysrYSktMT09PWlbYV0mJihpW2FdPTApO3N3aXRjaChpWzBdKXtjYXNlIDM4OnIuZmc9dGhpcy5fdXBkYXRlQXR0ckNvbG9yKHIuZmcsaVsxXSxpWzNdLGlbNF0saVs1XSk7YnJlYWs7Y2FzZSA0ODpyLmJnPXRoaXMuX3VwZGF0ZUF0dHJDb2xvcihyLmJnLGlbMV0saVszXSxpWzRdLGlbNV0pO2JyZWFrO2Nhc2UgNTg6ci5leHRlbmRlZD1yLmV4dGVuZGVkLmNsb25lKCksci5leHRlbmRlZC51bmRlcmxpbmVDb2xvcj10aGlzLl91cGRhdGVBdHRyQ29sb3Ioci5leHRlbmRlZC51bmRlcmxpbmVDb2xvcixpWzFdLGlbM10saVs0XSxpWzVdKX1yZXR1cm4gb30sdC5wcm90b3R5cGUuX3Byb2Nlc3NVbmRlcmxpbmU9ZnVuY3Rpb24oZSx0KXt0LmV4dGVuZGVkPXQuZXh0ZW5kZWQuY2xvbmUoKSwoIX5lfHxlPjUpJiYoZT0xKSx0LmV4dGVuZGVkLnVuZGVybGluZVN0eWxlPWUsdC5mZ3w9MjY4NDM1NDU2LDA9PT1lJiYodC5mZyY9LTI2ODQzNTQ1NyksdC51cGRhdGVFeHRlbmRlZCgpfSx0LnByb3RvdHlwZS5jaGFyQXR0cmlidXRlcz1mdW5jdGlvbihlKXtpZigxPT09ZS5sZW5ndGgmJjA9PT1lLnBhcmFtc1swXSlyZXR1cm4gdGhpcy5fY3VyQXR0ckRhdGEuZmc9Zi5ERUZBVUxUX0FUVFJfREFUQS5mZyx0aGlzLl9jdXJBdHRyRGF0YS5iZz1mLkRFRkFVTFRfQVRUUl9EQVRBLmJnLCEwO2Zvcih2YXIgdCxyPWUubGVuZ3RoLGk9dGhpcy5fY3VyQXR0ckRhdGEsbj0wO248cjtuKyspKHQ9ZS5wYXJhbXNbbl0pPj0zMCYmdDw9Mzc/KGkuZmcmPS01MDMzMTkwNCxpLmZnfD0xNjc3NzIxNnx0LTMwKTp0Pj00MCYmdDw9NDc/KGkuYmcmPS01MDMzMTkwNCxpLmJnfD0xNjc3NzIxNnx0LTQwKTp0Pj05MCYmdDw9OTc/KGkuZmcmPS01MDMzMTkwNCxpLmZnfD0xNjc3NzIyNHx0LTkwKTp0Pj0xMDAmJnQ8PTEwNz8oaS5iZyY9LTUwMzMxOTA0LGkuYmd8PTE2Nzc3MjI0fHQtMTAwKTowPT09dD8oaS5mZz1mLkRFRkFVTFRfQVRUUl9EQVRBLmZnLGkuYmc9Zi5ERUZBVUxUX0FUVFJfREFUQS5iZyk6MT09PXQ/aS5mZ3w9MTM0MjE3NzI4OjM9PT10P2kuYmd8PTY3MTA4ODY0OjQ9PT10PyhpLmZnfD0yNjg0MzU0NTYsdGhpcy5fcHJvY2Vzc1VuZGVybGluZShlLmhhc1N1YlBhcmFtcyhuKT9lLmdldFN1YlBhcmFtcyhuKVswXToxLGkpKTo1PT09dD9pLmZnfD01MzY4NzA5MTI6Nz09PXQ/aS5mZ3w9NjcxMDg4NjQ6OD09PXQ/aS5mZ3w9MTA3Mzc0MTgyNDo5PT09dD9pLmZnfD0yMTQ3NDgzNjQ4OjI9PT10P2kuYmd8PTEzNDIxNzcyODoyMT09PXQ/dGhpcy5fcHJvY2Vzc1VuZGVybGluZSgyLGkpOjIyPT09dD8oaS5mZyY9LTEzNDIxNzcyOSxpLmJnJj0tMTM0MjE3NzI5KToyMz09PXQ/aS5iZyY9LTY3MTA4ODY1OjI0PT09dD9pLmZnJj0tMjY4NDM1NDU3OjI1PT09dD9pLmZnJj0tNTM2ODcwOTEzOjI3PT09dD9pLmZnJj0tNjcxMDg4NjU6Mjg9PT10P2kuZmcmPS0xMDczNzQxODI1OjI5PT09dD9pLmZnJj0yMTQ3NDgzNjQ3OjM5PT09dD8oaS5mZyY9LTY3MTA4ODY0LGkuZmd8PTE2Nzc3MjE1JmYuREVGQVVMVF9BVFRSX0RBVEEuZmcpOjQ5PT09dD8oaS5iZyY9LTY3MTA4ODY0LGkuYmd8PTE2Nzc3MjE1JmYuREVGQVVMVF9BVFRSX0RBVEEuYmcpOjM4PT09dHx8NDg9PT10fHw1OD09PXQ/bis9dGhpcy5fZXh0cmFjdENvbG9yKGUsbixpKTo1OT09PXQ/KGkuZXh0ZW5kZWQ9aS5leHRlbmRlZC5jbG9uZSgpLGkuZXh0ZW5kZWQudW5kZXJsaW5lQ29sb3I9LTEsaS51cGRhdGVFeHRlbmRlZCgpKToxMDA9PT10PyhpLmZnJj0tNjcxMDg4NjQsaS5mZ3w9MTY3NzcyMTUmZi5ERUZBVUxUX0FUVFJfREFUQS5mZyxpLmJnJj0tNjcxMDg4NjQsaS5iZ3w9MTY3NzcyMTUmZi5ERUZBVUxUX0FUVFJfREFUQS5iZyk6dGhpcy5fbG9nU2VydmljZS5kZWJ1ZygiVW5rbm93biBTR1IgYXR0cmlidXRlOiAlZC4iLHQpO3JldHVybiEwfSx0LnByb3RvdHlwZS5kZXZpY2VTdGF0dXM9ZnVuY3Rpb24oZSl7c3dpdGNoKGUucGFyYW1zWzBdKXtjYXNlIDU6dGhpcy5fY29yZVNlcnZpY2UudHJpZ2dlckRhdGFFdmVudChzLkMwLkVTQysiWzBuIik7YnJlYWs7Y2FzZSA2OnZhciB0PXRoaXMuX2FjdGl2ZUJ1ZmZlci55KzEscj10aGlzLl9hY3RpdmVCdWZmZXIueCsxO3RoaXMuX2NvcmVTZXJ2aWNlLnRyaWdnZXJEYXRhRXZlbnQocy5DMC5FU0MrIlsiK3QrIjsiK3IrIlIiKX1yZXR1cm4hMH0sdC5wcm90b3R5cGUuZGV2aWNlU3RhdHVzUHJpdmF0ZT1mdW5jdGlvbihlKXtpZig2PT09ZS5wYXJhbXNbMF0pe3ZhciB0PXRoaXMuX2FjdGl2ZUJ1ZmZlci55KzEscj10aGlzLl9hY3RpdmVCdWZmZXIueCsxO3RoaXMuX2NvcmVTZXJ2aWNlLnRyaWdnZXJEYXRhRXZlbnQocy5DMC5FU0MrIls/Iit0KyI7IityKyJSIil9cmV0dXJuITB9LHQucHJvdG90eXBlLnNvZnRSZXNldD1mdW5jdGlvbihlKXtyZXR1cm4gdGhpcy5fY29yZVNlcnZpY2UuaXNDdXJzb3JIaWRkZW49ITEsdGhpcy5fb25SZXF1ZXN0U3luY1Njcm9sbEJhci5maXJlKCksdGhpcy5fYWN0aXZlQnVmZmVyLnNjcm9sbFRvcD0wLHRoaXMuX2FjdGl2ZUJ1ZmZlci5zY3JvbGxCb3R0b209dGhpcy5fYnVmZmVyU2VydmljZS5yb3dzLTEsdGhpcy5fY3VyQXR0ckRhdGE9Zi5ERUZBVUxUX0FUVFJfREFUQS5jbG9uZSgpLHRoaXMuX2NvcmVTZXJ2aWNlLnJlc2V0KCksdGhpcy5fY2hhcnNldFNlcnZpY2UucmVzZXQoKSx0aGlzLl9hY3RpdmVCdWZmZXIuc2F2ZWRYPTAsdGhpcy5fYWN0aXZlQnVmZmVyLnNhdmVkWT10aGlzLl9hY3RpdmVCdWZmZXIueWJhc2UsdGhpcy5fYWN0aXZlQnVmZmVyLnNhdmVkQ3VyQXR0ckRhdGEuZmc9dGhpcy5fY3VyQXR0ckRhdGEuZmcsdGhpcy5fYWN0aXZlQnVmZmVyLnNhdmVkQ3VyQXR0ckRhdGEuYmc9dGhpcy5fY3VyQXR0ckRhdGEuYmcsdGhpcy5fYWN0aXZlQnVmZmVyLnNhdmVkQ2hhcnNldD10aGlzLl9jaGFyc2V0U2VydmljZS5jaGFyc2V0LHRoaXMuX2NvcmVTZXJ2aWNlLmRlY1ByaXZhdGVNb2Rlcy5vcmlnaW49ITEsITB9LHQucHJvdG90eXBlLnNldEN1cnNvclN0eWxlPWZ1bmN0aW9uKGUpe3ZhciB0PWUucGFyYW1zWzBdfHwxO3N3aXRjaCh0KXtjYXNlIDE6Y2FzZSAyOnRoaXMuX29wdGlvbnNTZXJ2aWNlLm9wdGlvbnMuY3Vyc29yU3R5bGU9ImJsb2NrIjticmVhaztjYXNlIDM6Y2FzZSA0OnRoaXMuX29wdGlvbnNTZXJ2aWNlLm9wdGlvbnMuY3Vyc29yU3R5bGU9InVuZGVybGluZSI7YnJlYWs7Y2FzZSA1OmNhc2UgNjp0aGlzLl9vcHRpb25zU2VydmljZS5vcHRpb25zLmN1cnNvclN0eWxlPSJiYXIifXZhciByPXQlMj09MTtyZXR1cm4gdGhpcy5fb3B0aW9uc1NlcnZpY2Uub3B0aW9ucy5jdXJzb3JCbGluaz1yLCEwfSx0LnByb3RvdHlwZS5zZXRTY3JvbGxSZWdpb249ZnVuY3Rpb24oZSl7dmFyIHQscj1lLnBhcmFtc1swXXx8MTtyZXR1cm4oZS5sZW5ndGg8Mnx8KHQ9ZS5wYXJhbXNbMV0pPnRoaXMuX2J1ZmZlclNlcnZpY2Uucm93c3x8MD09PXQpJiYodD10aGlzLl9idWZmZXJTZXJ2aWNlLnJvd3MpLHQ+ciYmKHRoaXMuX2FjdGl2ZUJ1ZmZlci5zY3JvbGxUb3A9ci0xLHRoaXMuX2FjdGl2ZUJ1ZmZlci5zY3JvbGxCb3R0b209dC0xLHRoaXMuX3NldEN1cnNvcigwLDApKSwhMH0sdC5wcm90b3R5cGUud2luZG93T3B0aW9ucz1mdW5jdGlvbihlKXtpZighdyhlLnBhcmFtc1swXSx0aGlzLl9vcHRpb25zU2VydmljZS5vcHRpb25zLndpbmRvd09wdGlvbnMpKXJldHVybiEwO3ZhciB0PWUubGVuZ3RoPjE/ZS5wYXJhbXNbMV06MDtzd2l0Y2goZS5wYXJhbXNbMF0pe2Nhc2UgMTQ6MiE9PXQmJnRoaXMuX29uUmVxdWVzdFdpbmRvd3NPcHRpb25zUmVwb3J0LmZpcmUoby5HRVRfV0lOX1NJWkVfUElYRUxTKTticmVhaztjYXNlIDE2OnRoaXMuX29uUmVxdWVzdFdpbmRvd3NPcHRpb25zUmVwb3J0LmZpcmUoby5HRVRfQ0VMTF9TSVpFX1BJWEVMUyk7YnJlYWs7Y2FzZSAxODp0aGlzLl9idWZmZXJTZXJ2aWNlJiZ0aGlzLl9jb3JlU2VydmljZS50cmlnZ2VyRGF0YUV2ZW50KHMuQzAuRVNDKyJbODsiK3RoaXMuX2J1ZmZlclNlcnZpY2Uucm93cysiOyIrdGhpcy5fYnVmZmVyU2VydmljZS5jb2xzKyJ0Iik7YnJlYWs7Y2FzZSAyMjowIT09dCYmMiE9PXR8fCh0aGlzLl93aW5kb3dUaXRsZVN0YWNrLnB1c2godGhpcy5fd2luZG93VGl0bGUpLHRoaXMuX3dpbmRvd1RpdGxlU3RhY2subGVuZ3RoPjEwJiZ0aGlzLl93aW5kb3dUaXRsZVN0YWNrLnNoaWZ0KCkpLDAhPT10JiYxIT09dHx8KHRoaXMuX2ljb25OYW1lU3RhY2sucHVzaCh0aGlzLl9pY29uTmFtZSksdGhpcy5faWNvbk5hbWVTdGFjay5sZW5ndGg+MTAmJnRoaXMuX2ljb25OYW1lU3RhY2suc2hpZnQoKSk7YnJlYWs7Y2FzZSAyMzowIT09dCYmMiE9PXR8fHRoaXMuX3dpbmRvd1RpdGxlU3RhY2subGVuZ3RoJiZ0aGlzLnNldFRpdGxlKHRoaXMuX3dpbmRvd1RpdGxlU3RhY2sucG9wKCkpLDAhPT10JiYxIT09dHx8dGhpcy5faWNvbk5hbWVTdGFjay5sZW5ndGgmJnRoaXMuc2V0SWNvbk5hbWUodGhpcy5faWNvbk5hbWVTdGFjay5wb3AoKSl9cmV0dXJuITB9LHQucHJvdG90eXBlLnNhdmVDdXJzb3I9ZnVuY3Rpb24oZSl7cmV0dXJuIHRoaXMuX2FjdGl2ZUJ1ZmZlci5zYXZlZFg9dGhpcy5fYWN0aXZlQnVmZmVyLngsdGhpcy5fYWN0aXZlQnVmZmVyLnNhdmVkWT10aGlzLl9hY3RpdmVCdWZmZXIueWJhc2UrdGhpcy5fYWN0aXZlQnVmZmVyLnksdGhpcy5fYWN0aXZlQnVmZmVyLnNhdmVkQ3VyQXR0ckRhdGEuZmc9dGhpcy5fY3VyQXR0ckRhdGEuZmcsdGhpcy5fYWN0aXZlQnVmZmVyLnNhdmVkQ3VyQXR0ckRhdGEuYmc9dGhpcy5fY3VyQXR0ckRhdGEuYmcsdGhpcy5fYWN0aXZlQnVmZmVyLnNhdmVkQ2hhcnNldD10aGlzLl9jaGFyc2V0U2VydmljZS5jaGFyc2V0LCEwfSx0LnByb3RvdHlwZS5yZXN0b3JlQ3Vyc29yPWZ1bmN0aW9uKGUpe3JldHVybiB0aGlzLl9hY3RpdmVCdWZmZXIueD10aGlzLl9hY3RpdmVCdWZmZXIuc2F2ZWRYfHwwLHRoaXMuX2FjdGl2ZUJ1ZmZlci55PU1hdGgubWF4KHRoaXMuX2FjdGl2ZUJ1ZmZlci5zYXZlZFktdGhpcy5fYWN0aXZlQnVmZmVyLnliYXNlLDApLHRoaXMuX2N1ckF0dHJEYXRhLmZnPXRoaXMuX2FjdGl2ZUJ1ZmZlci5zYXZlZEN1ckF0dHJEYXRhLmZnLHRoaXMuX2N1ckF0dHJEYXRhLmJnPXRoaXMuX2FjdGl2ZUJ1ZmZlci5zYXZlZEN1ckF0dHJEYXRhLmJnLHRoaXMuX2NoYXJzZXRTZXJ2aWNlLmNoYXJzZXQ9dGhpcy5fc2F2ZWRDaGFyc2V0LHRoaXMuX2FjdGl2ZUJ1ZmZlci5zYXZlZENoYXJzZXQmJih0aGlzLl9jaGFyc2V0U2VydmljZS5jaGFyc2V0PXRoaXMuX2FjdGl2ZUJ1ZmZlci5zYXZlZENoYXJzZXQpLHRoaXMuX3Jlc3RyaWN0Q3Vyc29yKCksITB9LHQucHJvdG90eXBlLnNldFRpdGxlPWZ1bmN0aW9uKGUpe3JldHVybiB0aGlzLl93aW5kb3dUaXRsZT1lLHRoaXMuX29uVGl0bGVDaGFuZ2UuZmlyZShlKSwhMH0sdC5wcm90b3R5cGUuc2V0SWNvbk5hbWU9ZnVuY3Rpb24oZSl7cmV0dXJuIHRoaXMuX2ljb25OYW1lPWUsITB9LHQucHJvdG90eXBlLnNldE9yUmVwb3J0SW5kZXhlZENvbG9yPWZ1bmN0aW9uKGUpe2Zvcih2YXIgdD1bXSxyPWUuc3BsaXQoIjsiKTtyLmxlbmd0aD4xOyl7dmFyIGk9ci5zaGlmdCgpLG49ci5zaGlmdCgpO2lmKC9eXGQrJC8uZXhlYyhpKSl7dmFyIG89cGFyc2VJbnQoaSk7aWYoMDw9byYmbzwyNTYpaWYoIj8iPT09bil0LnB1c2goe3R5cGU6MCxpbmRleDpvfSk7ZWxzZXt2YXIgcz0oMCxiLnBhcnNlQ29sb3IpKG4pO3MmJnQucHVzaCh7dHlwZToxLGluZGV4Om8sY29sb3I6c30pfX19cmV0dXJuIHQubGVuZ3RoJiZ0aGlzLl9vbkNvbG9yLmZpcmUodCksITB9LHQucHJvdG90eXBlLl9zZXRPclJlcG9ydFNwZWNpYWxDb2xvcj1mdW5jdGlvbihlLHQpe2Zvcih2YXIgcj1lLnNwbGl0KCI7IiksaT0wO2k8ci5sZW5ndGgmJiEodD49dGhpcy5fc3BlY2lhbENvbG9ycy5sZW5ndGgpOysraSwrK3QpaWYoIj8iPT09cltpXSl0aGlzLl9vbkNvbG9yLmZpcmUoW3t0eXBlOjAsaW5kZXg6dGhpcy5fc3BlY2lhbENvbG9yc1t0XX1dKTtlbHNle3ZhciBuPSgwLGIucGFyc2VDb2xvcikocltpXSk7biYmdGhpcy5fb25Db2xvci5maXJlKFt7dHlwZToxLGluZGV4OnRoaXMuX3NwZWNpYWxDb2xvcnNbdF0sY29sb3I6bn1dKX1yZXR1cm4hMH0sdC5wcm90b3R5cGUuc2V0T3JSZXBvcnRGZ0NvbG9yPWZ1bmN0aW9uKGUpe3JldHVybiB0aGlzLl9zZXRPclJlcG9ydFNwZWNpYWxDb2xvcihlLDApfSx0LnByb3RvdHlwZS5zZXRPclJlcG9ydEJnQ29sb3I9ZnVuY3Rpb24oZSl7cmV0dXJuIHRoaXMuX3NldE9yUmVwb3J0U3BlY2lhbENvbG9yKGUsMSl9LHQucHJvdG90eXBlLnNldE9yUmVwb3J0Q3Vyc29yQ29sb3I9ZnVuY3Rpb24oZSl7cmV0dXJuIHRoaXMuX3NldE9yUmVwb3J0U3BlY2lhbENvbG9yKGUsMil9LHQucHJvdG90eXBlLnJlc3RvcmVJbmRleGVkQ29sb3I9ZnVuY3Rpb24oZSl7aWYoIWUpcmV0dXJuIHRoaXMuX29uQ29sb3IuZmlyZShbe3R5cGU6Mn1dKSwhMDtmb3IodmFyIHQ9W10scj1lLnNwbGl0KCI7IiksaT0wO2k8ci5sZW5ndGg7KytpKWlmKC9eXGQrJC8uZXhlYyhyW2ldKSl7dmFyIG49cGFyc2VJbnQocltpXSk7MDw9biYmbjwyNTYmJnQucHVzaCh7dHlwZToyLGluZGV4Om59KX1yZXR1cm4gdC5sZW5ndGgmJnRoaXMuX29uQ29sb3IuZmlyZSh0KSwhMH0sdC5wcm90b3R5cGUucmVzdG9yZUZnQ29sb3I9ZnVuY3Rpb24oZSl7cmV0dXJuIHRoaXMuX29uQ29sb3IuZmlyZShbe3R5cGU6MixpbmRleDoyNTZ9XSksITB9LHQucHJvdG90eXBlLnJlc3RvcmVCZ0NvbG9yPWZ1bmN0aW9uKGUpe3JldHVybiB0aGlzLl9vbkNvbG9yLmZpcmUoW3t0eXBlOjIsaW5kZXg6MjU3fV0pLCEwfSx0LnByb3RvdHlwZS5yZXN0b3JlQ3Vyc29yQ29sb3I9ZnVuY3Rpb24oZSl7cmV0dXJuIHRoaXMuX29uQ29sb3IuZmlyZShbe3R5cGU6MixpbmRleDoyNTh9XSksITB9LHQucHJvdG90eXBlLm5leHRMaW5lPWZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX2FjdGl2ZUJ1ZmZlci54PTAsdGhpcy5pbmRleCgpLCEwfSx0LnByb3RvdHlwZS5rZXlwYWRBcHBsaWNhdGlvbk1vZGU9ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fbG9nU2VydmljZS5kZWJ1ZygiU2VyaWFsIHBvcnQgcmVxdWVzdGVkIGFwcGxpY2F0aW9uIGtleXBhZC4iKSx0aGlzLl9jb3JlU2VydmljZS5kZWNQcml2YXRlTW9kZXMuYXBwbGljYXRpb25LZXlwYWQ9ITAsdGhpcy5fb25SZXF1ZXN0U3luY1Njcm9sbEJhci5maXJlKCksITB9LHQucHJvdG90eXBlLmtleXBhZE51bWVyaWNNb2RlPWZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX2xvZ1NlcnZpY2UuZGVidWcoIlN3aXRjaGluZyBiYWNrIHRvIG5vcm1hbCBrZXlwYWQuIiksdGhpcy5fY29yZVNlcnZpY2UuZGVjUHJpdmF0ZU1vZGVzLmFwcGxpY2F0aW9uS2V5cGFkPSExLHRoaXMuX29uUmVxdWVzdFN5bmNTY3JvbGxCYXIuZmlyZSgpLCEwfSx0LnByb3RvdHlwZS5zZWxlY3REZWZhdWx0Q2hhcnNldD1mdW5jdGlvbigpe3JldHVybiB0aGlzLl9jaGFyc2V0U2VydmljZS5zZXRnTGV2ZWwoMCksdGhpcy5fY2hhcnNldFNlcnZpY2Uuc2V0Z0NoYXJzZXQoMCxhLkRFRkFVTFRfQ0hBUlNFVCksITB9LHQucHJvdG90eXBlLnNlbGVjdENoYXJzZXQ9ZnVuY3Rpb24oZSl7cmV0dXJuIDIhPT1lLmxlbmd0aD8odGhpcy5zZWxlY3REZWZhdWx0Q2hhcnNldCgpLCEwKTooIi8iPT09ZVswXXx8dGhpcy5fY2hhcnNldFNlcnZpY2Uuc2V0Z0NoYXJzZXQoU1tlWzBdXSxhLkNIQVJTRVRTW2VbMV1dfHxhLkRFRkFVTFRfQ0hBUlNFVCksITApfSx0LnByb3RvdHlwZS5pbmRleD1mdW5jdGlvbigpe3JldHVybiB0aGlzLl9yZXN0cmljdEN1cnNvcigpLHRoaXMuX2FjdGl2ZUJ1ZmZlci55KyssdGhpcy5fYWN0aXZlQnVmZmVyLnk9PT10aGlzLl9hY3RpdmVCdWZmZXIuc2Nyb2xsQm90dG9tKzE/KHRoaXMuX2FjdGl2ZUJ1ZmZlci55LS0sdGhpcy5fYnVmZmVyU2VydmljZS5zY3JvbGwodGhpcy5fZXJhc2VBdHRyRGF0YSgpKSk6dGhpcy5fYWN0aXZlQnVmZmVyLnk+PXRoaXMuX2J1ZmZlclNlcnZpY2Uucm93cyYmKHRoaXMuX2FjdGl2ZUJ1ZmZlci55PXRoaXMuX2J1ZmZlclNlcnZpY2Uucm93cy0xKSx0aGlzLl9yZXN0cmljdEN1cnNvcigpLCEwfSx0LnByb3RvdHlwZS50YWJTZXQ9ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fYWN0aXZlQnVmZmVyLnRhYnNbdGhpcy5fYWN0aXZlQnVmZmVyLnhdPSEwLCEwfSx0LnByb3RvdHlwZS5yZXZlcnNlSW5kZXg9ZnVuY3Rpb24oKXtpZih0aGlzLl9yZXN0cmljdEN1cnNvcigpLHRoaXMuX2FjdGl2ZUJ1ZmZlci55PT09dGhpcy5fYWN0aXZlQnVmZmVyLnNjcm9sbFRvcCl7dmFyIGU9dGhpcy5fYWN0aXZlQnVmZmVyLnNjcm9sbEJvdHRvbS10aGlzLl9hY3RpdmVCdWZmZXIuc2Nyb2xsVG9wO3RoaXMuX2FjdGl2ZUJ1ZmZlci5saW5lcy5zaGlmdEVsZW1lbnRzKHRoaXMuX2FjdGl2ZUJ1ZmZlci55YmFzZSt0aGlzLl9hY3RpdmVCdWZmZXIueSxlLDEpLHRoaXMuX2FjdGl2ZUJ1ZmZlci5saW5lcy5zZXQodGhpcy5fYWN0aXZlQnVmZmVyLnliYXNlK3RoaXMuX2FjdGl2ZUJ1ZmZlci55LHRoaXMuX2FjdGl2ZUJ1ZmZlci5nZXRCbGFua0xpbmUodGhpcy5fZXJhc2VBdHRyRGF0YSgpKSksdGhpcy5fZGlydHlSb3dTZXJ2aWNlLm1hcmtSYW5nZURpcnR5KHRoaXMuX2FjdGl2ZUJ1ZmZlci5zY3JvbGxUb3AsdGhpcy5fYWN0aXZlQnVmZmVyLnNjcm9sbEJvdHRvbSl9ZWxzZSB0aGlzLl9hY3RpdmVCdWZmZXIueS0tLHRoaXMuX3Jlc3RyaWN0Q3Vyc29yKCk7cmV0dXJuITB9LHQucHJvdG90eXBlLmZ1bGxSZXNldD1mdW5jdGlvbigpe3JldHVybiB0aGlzLl9wYXJzZXIucmVzZXQoKSx0aGlzLl9vblJlcXVlc3RSZXNldC5maXJlKCksITB9LHQucHJvdG90eXBlLnJlc2V0PWZ1bmN0aW9uKCl7dGhpcy5fY3VyQXR0ckRhdGE9Zi5ERUZBVUxUX0FUVFJfREFUQS5jbG9uZSgpLHRoaXMuX2VyYXNlQXR0ckRhdGFJbnRlcm5hbD1mLkRFRkFVTFRfQVRUUl9EQVRBLmNsb25lKCl9LHQucHJvdG90eXBlLl9lcmFzZUF0dHJEYXRhPWZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX2VyYXNlQXR0ckRhdGFJbnRlcm5hbC5iZyY9LTY3MTA4ODY0LHRoaXMuX2VyYXNlQXR0ckRhdGFJbnRlcm5hbC5iZ3w9NjcxMDg4NjMmdGhpcy5fY3VyQXR0ckRhdGEuYmcsdGhpcy5fZXJhc2VBdHRyRGF0YUludGVybmFsfSx0LnByb3RvdHlwZS5zZXRnTGV2ZWw9ZnVuY3Rpb24oZSl7cmV0dXJuIHRoaXMuX2NoYXJzZXRTZXJ2aWNlLnNldGdMZXZlbChlKSwhMH0sdC5wcm90b3R5cGUuc2NyZWVuQWxpZ25tZW50UGF0dGVybj1mdW5jdGlvbigpe3ZhciBlPW5ldyBwLkNlbGxEYXRhO2UuY29udGVudD0xPDwyMnwiRSIuY2hhckNvZGVBdCgwKSxlLmZnPXRoaXMuX2N1ckF0dHJEYXRhLmZnLGUuYmc9dGhpcy5fY3VyQXR0ckRhdGEuYmcsdGhpcy5fc2V0Q3Vyc29yKDAsMCk7Zm9yKHZhciB0PTA7dDx0aGlzLl9idWZmZXJTZXJ2aWNlLnJvd3M7Kyt0KXt2YXIgcj10aGlzLl9hY3RpdmVCdWZmZXIueWJhc2UrdGhpcy5fYWN0aXZlQnVmZmVyLnkrdCxpPXRoaXMuX2FjdGl2ZUJ1ZmZlci5saW5lcy5nZXQocik7aSYmKGkuZmlsbChlKSxpLmlzV3JhcHBlZD0hMSl9cmV0dXJuIHRoaXMuX2RpcnR5Um93U2VydmljZS5tYXJrQWxsRGlydHkoKSx0aGlzLl9zZXRDdXJzb3IoMCwwKSwhMH0sdH0obC5EaXNwb3NhYmxlKTt0LklucHV0SGFuZGxlcj1FfSw4NDQ6KGUsdCk9PntPYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC5nZXREaXNwb3NlQXJyYXlEaXNwb3NhYmxlPXQuZGlzcG9zZUFycmF5PXQuRGlzcG9zYWJsZT12b2lkIDA7dmFyIHI9ZnVuY3Rpb24oKXtmdW5jdGlvbiBlKCl7dGhpcy5fZGlzcG9zYWJsZXM9W10sdGhpcy5faXNEaXNwb3NlZD0hMX1yZXR1cm4gZS5wcm90b3R5cGUuZGlzcG9zZT1mdW5jdGlvbigpe3RoaXMuX2lzRGlzcG9zZWQ9ITA7Zm9yKHZhciBlPTAsdD10aGlzLl9kaXNwb3NhYmxlcztlPHQubGVuZ3RoO2UrKyl0W2VdLmRpc3Bvc2UoKTt0aGlzLl9kaXNwb3NhYmxlcy5sZW5ndGg9MH0sZS5wcm90b3R5cGUucmVnaXN0ZXI9ZnVuY3Rpb24oZSl7cmV0dXJuIHRoaXMuX2Rpc3Bvc2FibGVzLnB1c2goZSksZX0sZS5wcm90b3R5cGUudW5yZWdpc3Rlcj1mdW5jdGlvbihlKXt2YXIgdD10aGlzLl9kaXNwb3NhYmxlcy5pbmRleE9mKGUpOy0xIT09dCYmdGhpcy5fZGlzcG9zYWJsZXMuc3BsaWNlKHQsMSl9LGV9KCk7ZnVuY3Rpb24gaShlKXtmb3IodmFyIHQ9MCxyPWU7dDxyLmxlbmd0aDt0Kyspclt0XS5kaXNwb3NlKCk7ZS5sZW5ndGg9MH10LkRpc3Bvc2FibGU9cix0LmRpc3Bvc2VBcnJheT1pLHQuZ2V0RGlzcG9zZUFycmF5RGlzcG9zYWJsZT1mdW5jdGlvbihlKXtyZXR1cm57ZGlzcG9zZTpmdW5jdGlvbigpe3JldHVybiBpKGUpfX19fSw2MTE0OihlLHQpPT57T2JqZWN0LmRlZmluZVByb3BlcnR5KHQsIl9fZXNNb2R1bGUiLHt2YWx1ZTohMH0pLHQuaXNMaW51eD10LmlzV2luZG93cz10LmlzSXBob25lPXQuaXNJcGFkPXQuaXNNYWM9dC5pc1NhZmFyaT10LmlzRmlyZWZveD12b2lkIDA7dmFyIHI9InVuZGVmaW5lZCI9PXR5cGVvZiBuYXZpZ2F0b3IsaT1yPyJub2RlIjpuYXZpZ2F0b3IudXNlckFnZW50LG49cj8ibm9kZSI6bmF2aWdhdG9yLnBsYXRmb3JtO3QuaXNGaXJlZm94PWkuaW5jbHVkZXMoIkZpcmVmb3giKSx0LmlzU2FmYXJpPS9eKCg/IWNocm9tZXxhbmRyb2lkKS4pKnNhZmFyaS9pLnRlc3QoaSksdC5pc01hYz1bIk1hY2ludG9zaCIsIk1hY0ludGVsIiwiTWFjUFBDIiwiTWFjNjhLIl0uaW5jbHVkZXMobiksdC5pc0lwYWQ9ImlQYWQiPT09bix0LmlzSXBob25lPSJpUGhvbmUiPT09bix0LmlzV2luZG93cz1bIldpbmRvd3MiLCJXaW4xNiIsIldpbjMyIiwiV2luQ0UiXS5pbmNsdWRlcyhuKSx0LmlzTGludXg9bi5pbmRleE9mKCJMaW51eCIpPj0wfSw4MjczOihlLHQpPT57ZnVuY3Rpb24gcihlLHQscixpKXtpZih2b2lkIDA9PT1yJiYocj0wKSx2b2lkIDA9PT1pJiYoaT1lLmxlbmd0aCkscj49ZS5sZW5ndGgpcmV0dXJuIGU7cj0oZS5sZW5ndGgrciklZS5sZW5ndGgsaT1pPj1lLmxlbmd0aD9lLmxlbmd0aDooZS5sZW5ndGgraSklZS5sZW5ndGg7Zm9yKHZhciBuPXI7bjxpOysrbillW25dPXQ7cmV0dXJuIGV9T2JqZWN0LmRlZmluZVByb3BlcnR5KHQsIl9fZXNNb2R1bGUiLHt2YWx1ZTohMH0pLHQuY29uY2F0PXQuZmlsbEZhbGxiYWNrPXQuZmlsbD12b2lkIDAsdC5maWxsPWZ1bmN0aW9uKGUsdCxpLG4pe3JldHVybiBlLmZpbGw/ZS5maWxsKHQsaSxuKTpyKGUsdCxpLG4pfSx0LmZpbGxGYWxsYmFjaz1yLHQuY29uY2F0PWZ1bmN0aW9uKGUsdCl7dmFyIHI9bmV3IGUuY29uc3RydWN0b3IoZS5sZW5ndGgrdC5sZW5ndGgpO3JldHVybiByLnNldChlKSxyLnNldCh0LGUubGVuZ3RoKSxyfX0sOTI4MjooZSx0LHIpPT57T2JqZWN0LmRlZmluZVByb3BlcnR5KHQsIl9fZXNNb2R1bGUiLHt2YWx1ZTohMH0pLHQudXBkYXRlV2luZG93c01vZGVXcmFwcGVkU3RhdGU9dm9pZCAwO3ZhciBpPXIoNjQzKTt0LnVwZGF0ZVdpbmRvd3NNb2RlV3JhcHBlZFN0YXRlPWZ1bmN0aW9uKGUpe3ZhciB0PWUuYnVmZmVyLmxpbmVzLmdldChlLmJ1ZmZlci55YmFzZStlLmJ1ZmZlci55LTEpLHI9bnVsbD09dD92b2lkIDA6dC5nZXQoZS5jb2xzLTEpLG49ZS5idWZmZXIubGluZXMuZ2V0KGUuYnVmZmVyLnliYXNlK2UuYnVmZmVyLnkpO24mJnImJihuLmlzV3JhcHBlZD1yW2kuQ0hBUl9EQVRBX0NPREVfSU5ERVhdIT09aS5OVUxMX0NFTExfQ09ERSYmcltpLkNIQVJfREFUQV9DT0RFX0lOREVYXSE9PWkuV0hJVEVTUEFDRV9DRUxMX0NPREUpfX0sMzczNDooZSx0KT0+e09iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LCJfX2VzTW9kdWxlIix7dmFsdWU6ITB9KSx0LkV4dGVuZGVkQXR0cnM9dC5BdHRyaWJ1dGVEYXRhPXZvaWQgMDt2YXIgcj1mdW5jdGlvbigpe2Z1bmN0aW9uIGUoKXt0aGlzLmZnPTAsdGhpcy5iZz0wLHRoaXMuZXh0ZW5kZWQ9bmV3IGl9cmV0dXJuIGUudG9Db2xvclJHQj1mdW5jdGlvbihlKXtyZXR1cm5bZT4+PjE2JjI1NSxlPj4+OCYyNTUsMjU1JmVdfSxlLmZyb21Db2xvclJHQj1mdW5jdGlvbihlKXtyZXR1cm4oMjU1JmVbMF0pPDwxNnwoMjU1JmVbMV0pPDw4fDI1NSZlWzJdfSxlLnByb3RvdHlwZS5jbG9uZT1mdW5jdGlvbigpe3ZhciB0PW5ldyBlO3JldHVybiB0LmZnPXRoaXMuZmcsdC5iZz10aGlzLmJnLHQuZXh0ZW5kZWQ9dGhpcy5leHRlbmRlZC5jbG9uZSgpLHR9LGUucHJvdG90eXBlLmlzSW52ZXJzZT1mdW5jdGlvbigpe3JldHVybiA2NzEwODg2NCZ0aGlzLmZnfSxlLnByb3RvdHlwZS5pc0JvbGQ9ZnVuY3Rpb24oKXtyZXR1cm4gMTM0MjE3NzI4JnRoaXMuZmd9LGUucHJvdG90eXBlLmlzVW5kZXJsaW5lPWZ1bmN0aW9uKCl7cmV0dXJuIDI2ODQzNTQ1NiZ0aGlzLmZnfSxlLnByb3RvdHlwZS5pc0JsaW5rPWZ1bmN0aW9uKCl7cmV0dXJuIDUzNjg3MDkxMiZ0aGlzLmZnfSxlLnByb3RvdHlwZS5pc0ludmlzaWJsZT1mdW5jdGlvbigpe3JldHVybiAxMDczNzQxODI0JnRoaXMuZmd9LGUucHJvdG90eXBlLmlzSXRhbGljPWZ1bmN0aW9uKCl7cmV0dXJuIDY3MTA4ODY0JnRoaXMuYmd9LGUucHJvdG90eXBlLmlzRGltPWZ1bmN0aW9uKCl7cmV0dXJuIDEzNDIxNzcyOCZ0aGlzLmJnfSxlLnByb3RvdHlwZS5pc1N0cmlrZXRocm91Z2g9ZnVuY3Rpb24oKXtyZXR1cm4gMjE0NzQ4MzY0OCZ0aGlzLmZnfSxlLnByb3RvdHlwZS5nZXRGZ0NvbG9yTW9kZT1mdW5jdGlvbigpe3JldHVybiA1MDMzMTY0OCZ0aGlzLmZnfSxlLnByb3RvdHlwZS5nZXRCZ0NvbG9yTW9kZT1mdW5jdGlvbigpe3JldHVybiA1MDMzMTY0OCZ0aGlzLmJnfSxlLnByb3RvdHlwZS5pc0ZnUkdCPWZ1bmN0aW9uKCl7cmV0dXJuIDUwMzMxNjQ4PT0oNTAzMzE2NDgmdGhpcy5mZyl9LGUucHJvdG90eXBlLmlzQmdSR0I9ZnVuY3Rpb24oKXtyZXR1cm4gNTAzMzE2NDg9PSg1MDMzMTY0OCZ0aGlzLmJnKX0sZS5wcm90b3R5cGUuaXNGZ1BhbGV0dGU9ZnVuY3Rpb24oKXtyZXR1cm4gMTY3NzcyMTY9PSg1MDMzMTY0OCZ0aGlzLmZnKXx8MzM1NTQ0MzI9PSg1MDMzMTY0OCZ0aGlzLmZnKX0sZS5wcm90b3R5cGUuaXNCZ1BhbGV0dGU9ZnVuY3Rpb24oKXtyZXR1cm4gMTY3NzcyMTY9PSg1MDMzMTY0OCZ0aGlzLmJnKXx8MzM1NTQ0MzI9PSg1MDMzMTY0OCZ0aGlzLmJnKX0sZS5wcm90b3R5cGUuaXNGZ0RlZmF1bHQ9ZnVuY3Rpb24oKXtyZXR1cm4gMD09KDUwMzMxNjQ4JnRoaXMuZmcpfSxlLnByb3RvdHlwZS5pc0JnRGVmYXVsdD1mdW5jdGlvbigpe3JldHVybiAwPT0oNTAzMzE2NDgmdGhpcy5iZyl9LGUucHJvdG90eXBlLmlzQXR0cmlidXRlRGVmYXVsdD1mdW5jdGlvbigpe3JldHVybiAwPT09dGhpcy5mZyYmMD09PXRoaXMuYmd9LGUucHJvdG90eXBlLmdldEZnQ29sb3I9ZnVuY3Rpb24oKXtzd2l0Y2goNTAzMzE2NDgmdGhpcy5mZyl7Y2FzZSAxNjc3NzIxNjpjYXNlIDMzNTU0NDMyOnJldHVybiAyNTUmdGhpcy5mZztjYXNlIDUwMzMxNjQ4OnJldHVybiAxNjc3NzIxNSZ0aGlzLmZnO2RlZmF1bHQ6cmV0dXJuLTF9fSxlLnByb3RvdHlwZS5nZXRCZ0NvbG9yPWZ1bmN0aW9uKCl7c3dpdGNoKDUwMzMxNjQ4JnRoaXMuYmcpe2Nhc2UgMTY3NzcyMTY6Y2FzZSAzMzU1NDQzMjpyZXR1cm4gMjU1JnRoaXMuYmc7Y2FzZSA1MDMzMTY0ODpyZXR1cm4gMTY3NzcyMTUmdGhpcy5iZztkZWZhdWx0OnJldHVybi0xfX0sZS5wcm90b3R5cGUuaGFzRXh0ZW5kZWRBdHRycz1mdW5jdGlvbigpe3JldHVybiAyNjg0MzU0NTYmdGhpcy5iZ30sZS5wcm90b3R5cGUudXBkYXRlRXh0ZW5kZWQ9ZnVuY3Rpb24oKXt0aGlzLmV4dGVuZGVkLmlzRW1wdHkoKT90aGlzLmJnJj0tMjY4NDM1NDU3OnRoaXMuYmd8PTI2ODQzNTQ1Nn0sZS5wcm90b3R5cGUuZ2V0VW5kZXJsaW5lQ29sb3I9ZnVuY3Rpb24oKXtpZigyNjg0MzU0NTYmdGhpcy5iZyYmfnRoaXMuZXh0ZW5kZWQudW5kZXJsaW5lQ29sb3Ipc3dpdGNoKDUwMzMxNjQ4JnRoaXMuZXh0ZW5kZWQudW5kZXJsaW5lQ29sb3Ipe2Nhc2UgMTY3NzcyMTY6Y2FzZSAzMzU1NDQzMjpyZXR1cm4gMjU1JnRoaXMuZXh0ZW5kZWQudW5kZXJsaW5lQ29sb3I7Y2FzZSA1MDMzMTY0ODpyZXR1cm4gMTY3NzcyMTUmdGhpcy5leHRlbmRlZC51bmRlcmxpbmVDb2xvcjtkZWZhdWx0OnJldHVybiB0aGlzLmdldEZnQ29sb3IoKX1yZXR1cm4gdGhpcy5nZXRGZ0NvbG9yKCl9LGUucHJvdG90eXBlLmdldFVuZGVybGluZUNvbG9yTW9kZT1mdW5jdGlvbigpe3JldHVybiAyNjg0MzU0NTYmdGhpcy5iZyYmfnRoaXMuZXh0ZW5kZWQudW5kZXJsaW5lQ29sb3I/NTAzMzE2NDgmdGhpcy5leHRlbmRlZC51bmRlcmxpbmVDb2xvcjp0aGlzLmdldEZnQ29sb3JNb2RlKCl9LGUucHJvdG90eXBlLmlzVW5kZXJsaW5lQ29sb3JSR0I9ZnVuY3Rpb24oKXtyZXR1cm4gMjY4NDM1NDU2JnRoaXMuYmcmJn50aGlzLmV4dGVuZGVkLnVuZGVybGluZUNvbG9yPzUwMzMxNjQ4PT0oNTAzMzE2NDgmdGhpcy5leHRlbmRlZC51bmRlcmxpbmVDb2xvcik6dGhpcy5pc0ZnUkdCKCl9LGUucHJvdG90eXBlLmlzVW5kZXJsaW5lQ29sb3JQYWxldHRlPWZ1bmN0aW9uKCl7cmV0dXJuIDI2ODQzNTQ1NiZ0aGlzLmJnJiZ+dGhpcy5leHRlbmRlZC51bmRlcmxpbmVDb2xvcj8xNjc3NzIxNj09KDUwMzMxNjQ4JnRoaXMuZXh0ZW5kZWQudW5kZXJsaW5lQ29sb3IpfHwzMzU1NDQzMj09KDUwMzMxNjQ4JnRoaXMuZXh0ZW5kZWQudW5kZXJsaW5lQ29sb3IpOnRoaXMuaXNGZ1BhbGV0dGUoKX0sZS5wcm90b3R5cGUuaXNVbmRlcmxpbmVDb2xvckRlZmF1bHQ9ZnVuY3Rpb24oKXtyZXR1cm4gMjY4NDM1NDU2JnRoaXMuYmcmJn50aGlzLmV4dGVuZGVkLnVuZGVybGluZUNvbG9yPzA9PSg1MDMzMTY0OCZ0aGlzLmV4dGVuZGVkLnVuZGVybGluZUNvbG9yKTp0aGlzLmlzRmdEZWZhdWx0KCl9LGUucHJvdG90eXBlLmdldFVuZGVybGluZVN0eWxlPWZ1bmN0aW9uKCl7cmV0dXJuIDI2ODQzNTQ1NiZ0aGlzLmZnPzI2ODQzNTQ1NiZ0aGlzLmJnP3RoaXMuZXh0ZW5kZWQudW5kZXJsaW5lU3R5bGU6MTowfSxlfSgpO3QuQXR0cmlidXRlRGF0YT1yO3ZhciBpPWZ1bmN0aW9uKCl7ZnVuY3Rpb24gZShlLHQpe3ZvaWQgMD09PWUmJihlPTApLHZvaWQgMD09PXQmJih0PS0xKSx0aGlzLnVuZGVybGluZVN0eWxlPWUsdGhpcy51bmRlcmxpbmVDb2xvcj10fXJldHVybiBlLnByb3RvdHlwZS5jbG9uZT1mdW5jdGlvbigpe3JldHVybiBuZXcgZSh0aGlzLnVuZGVybGluZVN0eWxlLHRoaXMudW5kZXJsaW5lQ29sb3IpfSxlLnByb3RvdHlwZS5pc0VtcHR5PWZ1bmN0aW9uKCl7cmV0dXJuIDA9PT10aGlzLnVuZGVybGluZVN0eWxlfSxlfSgpO3QuRXh0ZW5kZWRBdHRycz1pfSw5MDkyOihlLHQscik9PntPYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC5CdWZmZXJTdHJpbmdJdGVyYXRvcj10LkJ1ZmZlcj10Lk1BWF9CVUZGRVJfU0laRT12b2lkIDA7dmFyIGk9cig2MzQ5KSxuPXIoODQzNyksbz1yKDUxMSkscz1yKDY0MyksYT1yKDQ2MzQpLGM9cig0ODYzKSxsPXIoNzExNiksdT1yKDM3MzQpO3QuTUFYX0JVRkZFUl9TSVpFPTQyOTQ5NjcyOTU7dmFyIGg9ZnVuY3Rpb24oKXtmdW5jdGlvbiBlKGUsdCxyKXt0aGlzLl9oYXNTY3JvbGxiYWNrPWUsdGhpcy5fb3B0aW9uc1NlcnZpY2U9dCx0aGlzLl9idWZmZXJTZXJ2aWNlPXIsdGhpcy55ZGlzcD0wLHRoaXMueWJhc2U9MCx0aGlzLnk9MCx0aGlzLng9MCx0aGlzLnNhdmVkWT0wLHRoaXMuc2F2ZWRYPTAsdGhpcy5zYXZlZEN1ckF0dHJEYXRhPW4uREVGQVVMVF9BVFRSX0RBVEEuY2xvbmUoKSx0aGlzLnNhdmVkQ2hhcnNldD1sLkRFRkFVTFRfQ0hBUlNFVCx0aGlzLm1hcmtlcnM9W10sdGhpcy5fbnVsbENlbGw9by5DZWxsRGF0YS5mcm9tQ2hhckRhdGEoWzAscy5OVUxMX0NFTExfQ0hBUixzLk5VTExfQ0VMTF9XSURUSCxzLk5VTExfQ0VMTF9DT0RFXSksdGhpcy5fd2hpdGVzcGFjZUNlbGw9by5DZWxsRGF0YS5mcm9tQ2hhckRhdGEoWzAscy5XSElURVNQQUNFX0NFTExfQ0hBUixzLldISVRFU1BBQ0VfQ0VMTF9XSURUSCxzLldISVRFU1BBQ0VfQ0VMTF9DT0RFXSksdGhpcy5fY29scz10aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHMsdGhpcy5fcm93cz10aGlzLl9idWZmZXJTZXJ2aWNlLnJvd3MsdGhpcy5saW5lcz1uZXcgaS5DaXJjdWxhckxpc3QodGhpcy5fZ2V0Q29ycmVjdEJ1ZmZlckxlbmd0aCh0aGlzLl9yb3dzKSksdGhpcy5zY3JvbGxUb3A9MCx0aGlzLnNjcm9sbEJvdHRvbT10aGlzLl9yb3dzLTEsdGhpcy5zZXR1cFRhYlN0b3BzKCl9cmV0dXJuIGUucHJvdG90eXBlLmdldE51bGxDZWxsPWZ1bmN0aW9uKGUpe3JldHVybiBlPyh0aGlzLl9udWxsQ2VsbC5mZz1lLmZnLHRoaXMuX251bGxDZWxsLmJnPWUuYmcsdGhpcy5fbnVsbENlbGwuZXh0ZW5kZWQ9ZS5leHRlbmRlZCk6KHRoaXMuX251bGxDZWxsLmZnPTAsdGhpcy5fbnVsbENlbGwuYmc9MCx0aGlzLl9udWxsQ2VsbC5leHRlbmRlZD1uZXcgdS5FeHRlbmRlZEF0dHJzKSx0aGlzLl9udWxsQ2VsbH0sZS5wcm90b3R5cGUuZ2V0V2hpdGVzcGFjZUNlbGw9ZnVuY3Rpb24oZSl7cmV0dXJuIGU/KHRoaXMuX3doaXRlc3BhY2VDZWxsLmZnPWUuZmcsdGhpcy5fd2hpdGVzcGFjZUNlbGwuYmc9ZS5iZyx0aGlzLl93aGl0ZXNwYWNlQ2VsbC5leHRlbmRlZD1lLmV4dGVuZGVkKToodGhpcy5fd2hpdGVzcGFjZUNlbGwuZmc9MCx0aGlzLl93aGl0ZXNwYWNlQ2VsbC5iZz0wLHRoaXMuX3doaXRlc3BhY2VDZWxsLmV4dGVuZGVkPW5ldyB1LkV4dGVuZGVkQXR0cnMpLHRoaXMuX3doaXRlc3BhY2VDZWxsfSxlLnByb3RvdHlwZS5nZXRCbGFua0xpbmU9ZnVuY3Rpb24oZSx0KXtyZXR1cm4gbmV3IG4uQnVmZmVyTGluZSh0aGlzLl9idWZmZXJTZXJ2aWNlLmNvbHMsdGhpcy5nZXROdWxsQ2VsbChlKSx0KX0sT2JqZWN0LmRlZmluZVByb3BlcnR5KGUucHJvdG90eXBlLCJoYXNTY3JvbGxiYWNrIix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX2hhc1Njcm9sbGJhY2smJnRoaXMubGluZXMubWF4TGVuZ3RoPnRoaXMuX3Jvd3N9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KGUucHJvdG90eXBlLCJpc0N1cnNvckluVmlld3BvcnQiLHtnZXQ6ZnVuY3Rpb24oKXt2YXIgZT10aGlzLnliYXNlK3RoaXMueS10aGlzLnlkaXNwO3JldHVybiBlPj0wJiZlPHRoaXMuX3Jvd3N9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksZS5wcm90b3R5cGUuX2dldENvcnJlY3RCdWZmZXJMZW5ndGg9ZnVuY3Rpb24oZSl7aWYoIXRoaXMuX2hhc1Njcm9sbGJhY2spcmV0dXJuIGU7dmFyIHI9ZSt0aGlzLl9vcHRpb25zU2VydmljZS5vcHRpb25zLnNjcm9sbGJhY2s7cmV0dXJuIHI+dC5NQVhfQlVGRkVSX1NJWkU/dC5NQVhfQlVGRkVSX1NJWkU6cn0sZS5wcm90b3R5cGUuZmlsbFZpZXdwb3J0Um93cz1mdW5jdGlvbihlKXtpZigwPT09dGhpcy5saW5lcy5sZW5ndGgpe3ZvaWQgMD09PWUmJihlPW4uREVGQVVMVF9BVFRSX0RBVEEpO2Zvcih2YXIgdD10aGlzLl9yb3dzO3QtLTspdGhpcy5saW5lcy5wdXNoKHRoaXMuZ2V0QmxhbmtMaW5lKGUpKX19LGUucHJvdG90eXBlLmNsZWFyPWZ1bmN0aW9uKCl7dGhpcy55ZGlzcD0wLHRoaXMueWJhc2U9MCx0aGlzLnk9MCx0aGlzLng9MCx0aGlzLmxpbmVzPW5ldyBpLkNpcmN1bGFyTGlzdCh0aGlzLl9nZXRDb3JyZWN0QnVmZmVyTGVuZ3RoKHRoaXMuX3Jvd3MpKSx0aGlzLnNjcm9sbFRvcD0wLHRoaXMuc2Nyb2xsQm90dG9tPXRoaXMuX3Jvd3MtMSx0aGlzLnNldHVwVGFiU3RvcHMoKX0sZS5wcm90b3R5cGUucmVzaXplPWZ1bmN0aW9uKGUsdCl7dmFyIHI9dGhpcy5nZXROdWxsQ2VsbChuLkRFRkFVTFRfQVRUUl9EQVRBKSxpPXRoaXMuX2dldENvcnJlY3RCdWZmZXJMZW5ndGgodCk7aWYoaT50aGlzLmxpbmVzLm1heExlbmd0aCYmKHRoaXMubGluZXMubWF4TGVuZ3RoPWkpLHRoaXMubGluZXMubGVuZ3RoPjApe2lmKHRoaXMuX2NvbHM8ZSlmb3IodmFyIG89MDtvPHRoaXMubGluZXMubGVuZ3RoO28rKyl0aGlzLmxpbmVzLmdldChvKS5yZXNpemUoZSxyKTt2YXIgcz0wO2lmKHRoaXMuX3Jvd3M8dClmb3IodmFyIGE9dGhpcy5fcm93czthPHQ7YSsrKXRoaXMubGluZXMubGVuZ3RoPHQrdGhpcy55YmFzZSYmKHRoaXMuX29wdGlvbnNTZXJ2aWNlLm9wdGlvbnMud2luZG93c01vZGU/dGhpcy5saW5lcy5wdXNoKG5ldyBuLkJ1ZmZlckxpbmUoZSxyKSk6dGhpcy55YmFzZT4wJiZ0aGlzLmxpbmVzLmxlbmd0aDw9dGhpcy55YmFzZSt0aGlzLnkrcysxPyh0aGlzLnliYXNlLS0scysrLHRoaXMueWRpc3A+MCYmdGhpcy55ZGlzcC0tKTp0aGlzLmxpbmVzLnB1c2gobmV3IG4uQnVmZmVyTGluZShlLHIpKSk7ZWxzZSBmb3IoYT10aGlzLl9yb3dzO2E+dDthLS0pdGhpcy5saW5lcy5sZW5ndGg+dCt0aGlzLnliYXNlJiYodGhpcy5saW5lcy5sZW5ndGg+dGhpcy55YmFzZSt0aGlzLnkrMT90aGlzLmxpbmVzLnBvcCgpOih0aGlzLnliYXNlKyssdGhpcy55ZGlzcCsrKSk7aWYoaTx0aGlzLmxpbmVzLm1heExlbmd0aCl7dmFyIGM9dGhpcy5saW5lcy5sZW5ndGgtaTtjPjAmJih0aGlzLmxpbmVzLnRyaW1TdGFydChjKSx0aGlzLnliYXNlPU1hdGgubWF4KHRoaXMueWJhc2UtYywwKSx0aGlzLnlkaXNwPU1hdGgubWF4KHRoaXMueWRpc3AtYywwKSx0aGlzLnNhdmVkWT1NYXRoLm1heCh0aGlzLnNhdmVkWS1jLDApKSx0aGlzLmxpbmVzLm1heExlbmd0aD1pfXRoaXMueD1NYXRoLm1pbih0aGlzLngsZS0xKSx0aGlzLnk9TWF0aC5taW4odGhpcy55LHQtMSkscyYmKHRoaXMueSs9cyksdGhpcy5zYXZlZFg9TWF0aC5taW4odGhpcy5zYXZlZFgsZS0xKSx0aGlzLnNjcm9sbFRvcD0wfWlmKHRoaXMuc2Nyb2xsQm90dG9tPXQtMSx0aGlzLl9pc1JlZmxvd0VuYWJsZWQmJih0aGlzLl9yZWZsb3coZSx0KSx0aGlzLl9jb2xzPmUpKWZvcihvPTA7bzx0aGlzLmxpbmVzLmxlbmd0aDtvKyspdGhpcy5saW5lcy5nZXQobykucmVzaXplKGUscik7dGhpcy5fY29scz1lLHRoaXMuX3Jvd3M9dH0sT2JqZWN0LmRlZmluZVByb3BlcnR5KGUucHJvdG90eXBlLCJfaXNSZWZsb3dFbmFibGVkIix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX2hhc1Njcm9sbGJhY2smJiF0aGlzLl9vcHRpb25zU2VydmljZS5vcHRpb25zLndpbmRvd3NNb2RlfSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLGUucHJvdG90eXBlLl9yZWZsb3c9ZnVuY3Rpb24oZSx0KXt0aGlzLl9jb2xzIT09ZSYmKGU+dGhpcy5fY29scz90aGlzLl9yZWZsb3dMYXJnZXIoZSx0KTp0aGlzLl9yZWZsb3dTbWFsbGVyKGUsdCkpfSxlLnByb3RvdHlwZS5fcmVmbG93TGFyZ2VyPWZ1bmN0aW9uKGUsdCl7dmFyIHI9KDAsYS5yZWZsb3dMYXJnZXJHZXRMaW5lc1RvUmVtb3ZlKSh0aGlzLmxpbmVzLHRoaXMuX2NvbHMsZSx0aGlzLnliYXNlK3RoaXMueSx0aGlzLmdldE51bGxDZWxsKG4uREVGQVVMVF9BVFRSX0RBVEEpKTtpZihyLmxlbmd0aD4wKXt2YXIgaT0oMCxhLnJlZmxvd0xhcmdlckNyZWF0ZU5ld0xheW91dCkodGhpcy5saW5lcyxyKTsoMCxhLnJlZmxvd0xhcmdlckFwcGx5TmV3TGF5b3V0KSh0aGlzLmxpbmVzLGkubGF5b3V0KSx0aGlzLl9yZWZsb3dMYXJnZXJBZGp1c3RWaWV3cG9ydChlLHQsaS5jb3VudFJlbW92ZWQpfX0sZS5wcm90b3R5cGUuX3JlZmxvd0xhcmdlckFkanVzdFZpZXdwb3J0PWZ1bmN0aW9uKGUsdCxyKXtmb3IodmFyIGk9dGhpcy5nZXROdWxsQ2VsbChuLkRFRkFVTFRfQVRUUl9EQVRBKSxvPXI7by0tID4wOykwPT09dGhpcy55YmFzZT8odGhpcy55PjAmJnRoaXMueS0tLHRoaXMubGluZXMubGVuZ3RoPHQmJnRoaXMubGluZXMucHVzaChuZXcgbi5CdWZmZXJMaW5lKGUsaSkpKToodGhpcy55ZGlzcD09PXRoaXMueWJhc2UmJnRoaXMueWRpc3AtLSx0aGlzLnliYXNlLS0pO3RoaXMuc2F2ZWRZPU1hdGgubWF4KHRoaXMuc2F2ZWRZLXIsMCl9LGUucHJvdG90eXBlLl9yZWZsb3dTbWFsbGVyPWZ1bmN0aW9uKGUsdCl7Zm9yKHZhciByPXRoaXMuZ2V0TnVsbENlbGwobi5ERUZBVUxUX0FUVFJfREFUQSksaT1bXSxvPTAscz10aGlzLmxpbmVzLmxlbmd0aC0xO3M+PTA7cy0tKXt2YXIgYz10aGlzLmxpbmVzLmdldChzKTtpZighKCFjfHwhYy5pc1dyYXBwZWQmJmMuZ2V0VHJpbW1lZExlbmd0aCgpPD1lKSl7Zm9yKHZhciBsPVtjXTtjLmlzV3JhcHBlZCYmcz4wOyljPXRoaXMubGluZXMuZ2V0KC0tcyksbC51bnNoaWZ0KGMpO3ZhciB1PXRoaXMueWJhc2UrdGhpcy55O2lmKCEodT49cyYmdTxzK2wubGVuZ3RoKSl7dmFyIGgsZj1sW2wubGVuZ3RoLTFdLmdldFRyaW1tZWRMZW5ndGgoKSxfPSgwLGEucmVmbG93U21hbGxlckdldE5ld0xpbmVMZW5ndGhzKShsLHRoaXMuX2NvbHMsZSksZD1fLmxlbmd0aC1sLmxlbmd0aDtoPTA9PT10aGlzLnliYXNlJiZ0aGlzLnkhPT10aGlzLmxpbmVzLmxlbmd0aC0xP01hdGgubWF4KDAsdGhpcy55LXRoaXMubGluZXMubWF4TGVuZ3RoK2QpOk1hdGgubWF4KDAsdGhpcy5saW5lcy5sZW5ndGgtdGhpcy5saW5lcy5tYXhMZW5ndGgrZCk7Zm9yKHZhciBwPVtdLHY9MDt2PGQ7disrKXt2YXIgZz10aGlzLmdldEJsYW5rTGluZShuLkRFRkFVTFRfQVRUUl9EQVRBLCEwKTtwLnB1c2goZyl9cC5sZW5ndGg+MCYmKGkucHVzaCh7c3RhcnQ6cytsLmxlbmd0aCtvLG5ld0xpbmVzOnB9KSxvKz1wLmxlbmd0aCksbC5wdXNoLmFwcGx5KGwscCk7dmFyIHk9Xy5sZW5ndGgtMSxtPV9beV07MD09PW0mJihtPV9bLS15XSk7Zm9yKHZhciBiPWwubGVuZ3RoLWQtMSxTPWY7Yj49MDspe3ZhciBDPU1hdGgubWluKFMsbSk7aWYobFt5XS5jb3B5Q2VsbHNGcm9tKGxbYl0sUy1DLG0tQyxDLCEwKSwwPT0obS09QykmJihtPV9bLS15XSksMD09KFMtPUMpKXtiLS07dmFyIHc9TWF0aC5tYXgoYiwwKTtTPSgwLGEuZ2V0V3JhcHBlZExpbmVUcmltbWVkTGVuZ3RoKShsLHcsdGhpcy5fY29scyl9fWZvcih2PTA7djxsLmxlbmd0aDt2KyspX1t2XTxlJiZsW3ZdLnNldENlbGwoX1t2XSxyKTtmb3IodmFyIEw9ZC1oO0wtLSA+MDspMD09PXRoaXMueWJhc2U/dGhpcy55PHQtMT8odGhpcy55KyssdGhpcy5saW5lcy5wb3AoKSk6KHRoaXMueWJhc2UrKyx0aGlzLnlkaXNwKyspOnRoaXMueWJhc2U8TWF0aC5taW4odGhpcy5saW5lcy5tYXhMZW5ndGgsdGhpcy5saW5lcy5sZW5ndGgrbyktdCYmKHRoaXMueWJhc2U9PT10aGlzLnlkaXNwJiZ0aGlzLnlkaXNwKyssdGhpcy55YmFzZSsrKTt0aGlzLnNhdmVkWT1NYXRoLm1pbih0aGlzLnNhdmVkWStkLHRoaXMueWJhc2UrdC0xKX19fWlmKGkubGVuZ3RoPjApe3ZhciBFPVtdLHg9W107Zm9yKHY9MDt2PHRoaXMubGluZXMubGVuZ3RoO3YrKyl4LnB1c2godGhpcy5saW5lcy5nZXQodikpO3ZhciBBPXRoaXMubGluZXMubGVuZ3RoLGs9QS0xLE09MCxSPWlbTV07dGhpcy5saW5lcy5sZW5ndGg9TWF0aC5taW4odGhpcy5saW5lcy5tYXhMZW5ndGgsdGhpcy5saW5lcy5sZW5ndGgrbyk7dmFyIFQ9MDtmb3Iodj1NYXRoLm1pbih0aGlzLmxpbmVzLm1heExlbmd0aC0xLEErby0xKTt2Pj0wO3YtLSlpZihSJiZSLnN0YXJ0PmsrVCl7Zm9yKHZhciBPPVIubmV3TGluZXMubGVuZ3RoLTE7Tz49MDtPLS0pdGhpcy5saW5lcy5zZXQodi0tLFIubmV3TGluZXNbT10pO3YrKyxFLnB1c2goe2luZGV4OmsrMSxhbW91bnQ6Ui5uZXdMaW5lcy5sZW5ndGh9KSxUKz1SLm5ld0xpbmVzLmxlbmd0aCxSPWlbKytNXX1lbHNlIHRoaXMubGluZXMuc2V0KHYseFtrLS1dKTt2YXIgQj0wO2Zvcih2PUUubGVuZ3RoLTE7dj49MDt2LS0pRVt2XS5pbmRleCs9Qix0aGlzLmxpbmVzLm9uSW5zZXJ0RW1pdHRlci5maXJlKEVbdl0pLEIrPUVbdl0uYW1vdW50O3ZhciBEPU1hdGgubWF4KDAsQStvLXRoaXMubGluZXMubWF4TGVuZ3RoKTtEPjAmJnRoaXMubGluZXMub25UcmltRW1pdHRlci5maXJlKEQpfX0sZS5wcm90b3R5cGUuc3RyaW5nSW5kZXhUb0J1ZmZlckluZGV4PWZ1bmN0aW9uKGUsdCxyKXtmb3Iodm9pZCAwPT09ciYmKHI9ITEpO3Q7KXt2YXIgaT10aGlzLmxpbmVzLmdldChlKTtpZighaSlyZXR1cm5bLTEsLTFdO2Zvcih2YXIgbj1yP2kuZ2V0VHJpbW1lZExlbmd0aCgpOmkubGVuZ3RoLG89MDtvPG47KytvKWlmKGkuZ2V0KG8pW3MuQ0hBUl9EQVRBX1dJRFRIX0lOREVYXSYmKHQtPWkuZ2V0KG8pW3MuQ0hBUl9EQVRBX0NIQVJfSU5ERVhdLmxlbmd0aHx8MSksdDwwKXJldHVybltlLG9dO2UrK31yZXR1cm5bZSwwXX0sZS5wcm90b3R5cGUudHJhbnNsYXRlQnVmZmVyTGluZVRvU3RyaW5nPWZ1bmN0aW9uKGUsdCxyLGkpe3ZvaWQgMD09PXImJihyPTApO3ZhciBuPXRoaXMubGluZXMuZ2V0KGUpO3JldHVybiBuP24udHJhbnNsYXRlVG9TdHJpbmcodCxyLGkpOiIifSxlLnByb3RvdHlwZS5nZXRXcmFwcGVkUmFuZ2VGb3JMaW5lPWZ1bmN0aW9uKGUpe2Zvcih2YXIgdD1lLHI9ZTt0PjAmJnRoaXMubGluZXMuZ2V0KHQpLmlzV3JhcHBlZDspdC0tO2Zvcig7cisxPHRoaXMubGluZXMubGVuZ3RoJiZ0aGlzLmxpbmVzLmdldChyKzEpLmlzV3JhcHBlZDspcisrO3JldHVybntmaXJzdDp0LGxhc3Q6cn19LGUucHJvdG90eXBlLnNldHVwVGFiU3RvcHM9ZnVuY3Rpb24oZSl7Zm9yKG51bGwhPWU/dGhpcy50YWJzW2VdfHwoZT10aGlzLnByZXZTdG9wKGUpKToodGhpcy50YWJzPXt9LGU9MCk7ZTx0aGlzLl9jb2xzO2UrPXRoaXMuX29wdGlvbnNTZXJ2aWNlLm9wdGlvbnMudGFiU3RvcFdpZHRoKXRoaXMudGFic1tlXT0hMH0sZS5wcm90b3R5cGUucHJldlN0b3A9ZnVuY3Rpb24oZSl7Zm9yKG51bGw9PWUmJihlPXRoaXMueCk7IXRoaXMudGFic1stLWVdJiZlPjA7KTtyZXR1cm4gZT49dGhpcy5fY29scz90aGlzLl9jb2xzLTE6ZTwwPzA6ZX0sZS5wcm90b3R5cGUubmV4dFN0b3A9ZnVuY3Rpb24oZSl7Zm9yKG51bGw9PWUmJihlPXRoaXMueCk7IXRoaXMudGFic1srK2VdJiZlPHRoaXMuX2NvbHM7KTtyZXR1cm4gZT49dGhpcy5fY29scz90aGlzLl9jb2xzLTE6ZTwwPzA6ZX0sZS5wcm90b3R5cGUuYWRkTWFya2VyPWZ1bmN0aW9uKGUpe3ZhciB0PXRoaXMscj1uZXcgYy5NYXJrZXIoZSk7cmV0dXJuIHRoaXMubWFya2Vycy5wdXNoKHIpLHIucmVnaXN0ZXIodGhpcy5saW5lcy5vblRyaW0oKGZ1bmN0aW9uKGUpe3IubGluZS09ZSxyLmxpbmU8MCYmci5kaXNwb3NlKCl9KSkpLHIucmVnaXN0ZXIodGhpcy5saW5lcy5vbkluc2VydCgoZnVuY3Rpb24oZSl7ci5saW5lPj1lLmluZGV4JiYoci5saW5lKz1lLmFtb3VudCl9KSkpLHIucmVnaXN0ZXIodGhpcy5saW5lcy5vbkRlbGV0ZSgoZnVuY3Rpb24oZSl7ci5saW5lPj1lLmluZGV4JiZyLmxpbmU8ZS5pbmRleCtlLmFtb3VudCYmci5kaXNwb3NlKCksci5saW5lPmUuaW5kZXgmJihyLmxpbmUtPWUuYW1vdW50KX0pKSksci5yZWdpc3RlcihyLm9uRGlzcG9zZSgoZnVuY3Rpb24oKXtyZXR1cm4gdC5fcmVtb3ZlTWFya2VyKHIpfSkpKSxyfSxlLnByb3RvdHlwZS5fcmVtb3ZlTWFya2VyPWZ1bmN0aW9uKGUpe3RoaXMubWFya2Vycy5zcGxpY2UodGhpcy5tYXJrZXJzLmluZGV4T2YoZSksMSl9LGUucHJvdG90eXBlLml0ZXJhdG9yPWZ1bmN0aW9uKGUsdCxyLGksbil7cmV0dXJuIG5ldyBmKHRoaXMsZSx0LHIsaSxuKX0sZX0oKTt0LkJ1ZmZlcj1oO3ZhciBmPWZ1bmN0aW9uKCl7ZnVuY3Rpb24gZShlLHQscixpLG4sbyl7dm9pZCAwPT09ciYmKHI9MCksdm9pZCAwPT09aSYmKGk9ZS5saW5lcy5sZW5ndGgpLHZvaWQgMD09PW4mJihuPTApLHZvaWQgMD09PW8mJihvPTApLHRoaXMuX2J1ZmZlcj1lLHRoaXMuX3RyaW1SaWdodD10LHRoaXMuX3N0YXJ0SW5kZXg9cix0aGlzLl9lbmRJbmRleD1pLHRoaXMuX3N0YXJ0T3ZlcnNjYW49bix0aGlzLl9lbmRPdmVyc2Nhbj1vLHRoaXMuX3N0YXJ0SW5kZXg8MCYmKHRoaXMuX3N0YXJ0SW5kZXg9MCksdGhpcy5fZW5kSW5kZXg+dGhpcy5fYnVmZmVyLmxpbmVzLmxlbmd0aCYmKHRoaXMuX2VuZEluZGV4PXRoaXMuX2J1ZmZlci5saW5lcy5sZW5ndGgpLHRoaXMuX2N1cnJlbnQ9dGhpcy5fc3RhcnRJbmRleH1yZXR1cm4gZS5wcm90b3R5cGUuaGFzTmV4dD1mdW5jdGlvbigpe3JldHVybiB0aGlzLl9jdXJyZW50PHRoaXMuX2VuZEluZGV4fSxlLnByb3RvdHlwZS5uZXh0PWZ1bmN0aW9uKCl7dmFyIGU9dGhpcy5fYnVmZmVyLmdldFdyYXBwZWRSYW5nZUZvckxpbmUodGhpcy5fY3VycmVudCk7ZS5maXJzdDx0aGlzLl9zdGFydEluZGV4LXRoaXMuX3N0YXJ0T3ZlcnNjYW4mJihlLmZpcnN0PXRoaXMuX3N0YXJ0SW5kZXgtdGhpcy5fc3RhcnRPdmVyc2NhbiksZS5sYXN0PnRoaXMuX2VuZEluZGV4K3RoaXMuX2VuZE92ZXJzY2FuJiYoZS5sYXN0PXRoaXMuX2VuZEluZGV4K3RoaXMuX2VuZE92ZXJzY2FuKSxlLmZpcnN0PU1hdGgubWF4KGUuZmlyc3QsMCksZS5sYXN0PU1hdGgubWluKGUubGFzdCx0aGlzLl9idWZmZXIubGluZXMubGVuZ3RoKTtmb3IodmFyIHQ9IiIscj1lLmZpcnN0O3I8PWUubGFzdDsrK3IpdCs9dGhpcy5fYnVmZmVyLnRyYW5zbGF0ZUJ1ZmZlckxpbmVUb1N0cmluZyhyLHRoaXMuX3RyaW1SaWdodCk7cmV0dXJuIHRoaXMuX2N1cnJlbnQ9ZS5sYXN0KzEse3JhbmdlOmUsY29udGVudDp0fX0sZX0oKTt0LkJ1ZmZlclN0cmluZ0l0ZXJhdG9yPWZ9LDg0Mzc6KGUsdCxyKT0+e09iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LCJfX2VzTW9kdWxlIix7dmFsdWU6ITB9KSx0LkJ1ZmZlckxpbmU9dC5ERUZBVUxUX0FUVFJfREFUQT12b2lkIDA7dmFyIGk9cig0ODIpLG49cig2NDMpLG89cig1MTEpLHM9cigzNzM0KTt0LkRFRkFVTFRfQVRUUl9EQVRBPU9iamVjdC5mcmVlemUobmV3IHMuQXR0cmlidXRlRGF0YSk7dmFyIGE9ZnVuY3Rpb24oKXtmdW5jdGlvbiBlKGUsdCxyKXt2b2lkIDA9PT1yJiYocj0hMSksdGhpcy5pc1dyYXBwZWQ9cix0aGlzLl9jb21iaW5lZD17fSx0aGlzLl9leHRlbmRlZEF0dHJzPXt9LHRoaXMuX2RhdGE9bmV3IFVpbnQzMkFycmF5KDMqZSk7Zm9yKHZhciBpPXR8fG8uQ2VsbERhdGEuZnJvbUNoYXJEYXRhKFswLG4uTlVMTF9DRUxMX0NIQVIsbi5OVUxMX0NFTExfV0lEVEgsbi5OVUxMX0NFTExfQ09ERV0pLHM9MDtzPGU7KytzKXRoaXMuc2V0Q2VsbChzLGkpO3RoaXMubGVuZ3RoPWV9cmV0dXJuIGUucHJvdG90eXBlLmdldD1mdW5jdGlvbihlKXt2YXIgdD10aGlzLl9kYXRhWzMqZSswXSxyPTIwOTcxNTEmdDtyZXR1cm5bdGhpcy5fZGF0YVszKmUrMV0sMjA5NzE1MiZ0P3RoaXMuX2NvbWJpbmVkW2VdOnI/KDAsaS5zdHJpbmdGcm9tQ29kZVBvaW50KShyKToiIix0Pj4yMiwyMDk3MTUyJnQ/dGhpcy5fY29tYmluZWRbZV0uY2hhckNvZGVBdCh0aGlzLl9jb21iaW5lZFtlXS5sZW5ndGgtMSk6cl19LGUucHJvdG90eXBlLnNldD1mdW5jdGlvbihlLHQpe3RoaXMuX2RhdGFbMyplKzFdPXRbbi5DSEFSX0RBVEFfQVRUUl9JTkRFWF0sdFtuLkNIQVJfREFUQV9DSEFSX0lOREVYXS5sZW5ndGg+MT8odGhpcy5fY29tYmluZWRbZV09dFsxXSx0aGlzLl9kYXRhWzMqZSswXT0yMDk3MTUyfGV8dFtuLkNIQVJfREFUQV9XSURUSF9JTkRFWF08PDIyKTp0aGlzLl9kYXRhWzMqZSswXT10W24uQ0hBUl9EQVRBX0NIQVJfSU5ERVhdLmNoYXJDb2RlQXQoMCl8dFtuLkNIQVJfREFUQV9XSURUSF9JTkRFWF08PDIyfSxlLnByb3RvdHlwZS5nZXRXaWR0aD1mdW5jdGlvbihlKXtyZXR1cm4gdGhpcy5fZGF0YVszKmUrMF0+PjIyfSxlLnByb3RvdHlwZS5oYXNXaWR0aD1mdW5jdGlvbihlKXtyZXR1cm4gMTI1ODI5MTImdGhpcy5fZGF0YVszKmUrMF19LGUucHJvdG90eXBlLmdldEZnPWZ1bmN0aW9uKGUpe3JldHVybiB0aGlzLl9kYXRhWzMqZSsxXX0sZS5wcm90b3R5cGUuZ2V0Qmc9ZnVuY3Rpb24oZSl7cmV0dXJuIHRoaXMuX2RhdGFbMyplKzJdfSxlLnByb3RvdHlwZS5oYXNDb250ZW50PWZ1bmN0aW9uKGUpe3JldHVybiA0MTk0MzAzJnRoaXMuX2RhdGFbMyplKzBdfSxlLnByb3RvdHlwZS5nZXRDb2RlUG9pbnQ9ZnVuY3Rpb24oZSl7dmFyIHQ9dGhpcy5fZGF0YVszKmUrMF07cmV0dXJuIDIwOTcxNTImdD90aGlzLl9jb21iaW5lZFtlXS5jaGFyQ29kZUF0KHRoaXMuX2NvbWJpbmVkW2VdLmxlbmd0aC0xKToyMDk3MTUxJnR9LGUucHJvdG90eXBlLmlzQ29tYmluZWQ9ZnVuY3Rpb24oZSl7cmV0dXJuIDIwOTcxNTImdGhpcy5fZGF0YVszKmUrMF19LGUucHJvdG90eXBlLmdldFN0cmluZz1mdW5jdGlvbihlKXt2YXIgdD10aGlzLl9kYXRhWzMqZSswXTtyZXR1cm4gMjA5NzE1MiZ0P3RoaXMuX2NvbWJpbmVkW2VdOjIwOTcxNTEmdD8oMCxpLnN0cmluZ0Zyb21Db2RlUG9pbnQpKDIwOTcxNTEmdCk6IiJ9LGUucHJvdG90eXBlLmxvYWRDZWxsPWZ1bmN0aW9uKGUsdCl7dmFyIHI9MyplO3JldHVybiB0LmNvbnRlbnQ9dGhpcy5fZGF0YVtyKzBdLHQuZmc9dGhpcy5fZGF0YVtyKzFdLHQuYmc9dGhpcy5fZGF0YVtyKzJdLDIwOTcxNTImdC5jb250ZW50JiYodC5jb21iaW5lZERhdGE9dGhpcy5fY29tYmluZWRbZV0pLDI2ODQzNTQ1NiZ0LmJnJiYodC5leHRlbmRlZD10aGlzLl9leHRlbmRlZEF0dHJzW2VdKSx0fSxlLnByb3RvdHlwZS5zZXRDZWxsPWZ1bmN0aW9uKGUsdCl7MjA5NzE1MiZ0LmNvbnRlbnQmJih0aGlzLl9jb21iaW5lZFtlXT10LmNvbWJpbmVkRGF0YSksMjY4NDM1NDU2JnQuYmcmJih0aGlzLl9leHRlbmRlZEF0dHJzW2VdPXQuZXh0ZW5kZWQpLHRoaXMuX2RhdGFbMyplKzBdPXQuY29udGVudCx0aGlzLl9kYXRhWzMqZSsxXT10LmZnLHRoaXMuX2RhdGFbMyplKzJdPXQuYmd9LGUucHJvdG90eXBlLnNldENlbGxGcm9tQ29kZVBvaW50PWZ1bmN0aW9uKGUsdCxyLGksbixvKXsyNjg0MzU0NTYmbiYmKHRoaXMuX2V4dGVuZGVkQXR0cnNbZV09byksdGhpcy5fZGF0YVszKmUrMF09dHxyPDwyMix0aGlzLl9kYXRhWzMqZSsxXT1pLHRoaXMuX2RhdGFbMyplKzJdPW59LGUucHJvdG90eXBlLmFkZENvZGVwb2ludFRvQ2VsbD1mdW5jdGlvbihlLHQpe3ZhciByPXRoaXMuX2RhdGFbMyplKzBdOzIwOTcxNTImcj90aGlzLl9jb21iaW5lZFtlXSs9KDAsaS5zdHJpbmdGcm9tQ29kZVBvaW50KSh0KTooMjA5NzE1MSZyPyh0aGlzLl9jb21iaW5lZFtlXT0oMCxpLnN0cmluZ0Zyb21Db2RlUG9pbnQpKDIwOTcxNTEmcikrKDAsaS5zdHJpbmdGcm9tQ29kZVBvaW50KSh0KSxyJj0tMjA5NzE1MixyfD0yMDk3MTUyKTpyPXR8MTw8MjIsdGhpcy5fZGF0YVszKmUrMF09cil9LGUucHJvdG90eXBlLmluc2VydENlbGxzPWZ1bmN0aW9uKGUsdCxyLGkpe2lmKChlJT10aGlzLmxlbmd0aCkmJjI9PT10aGlzLmdldFdpZHRoKGUtMSkmJnRoaXMuc2V0Q2VsbEZyb21Db2RlUG9pbnQoZS0xLDAsMSwobnVsbD09aT92b2lkIDA6aS5mZyl8fDAsKG51bGw9PWk/dm9pZCAwOmkuYmcpfHwwLChudWxsPT1pP3ZvaWQgMDppLmV4dGVuZGVkKXx8bmV3IHMuRXh0ZW5kZWRBdHRycyksdDx0aGlzLmxlbmd0aC1lKXtmb3IodmFyIG49bmV3IG8uQ2VsbERhdGEsYT10aGlzLmxlbmd0aC1lLXQtMTthPj0wOy0tYSl0aGlzLnNldENlbGwoZSt0K2EsdGhpcy5sb2FkQ2VsbChlK2EsbikpO2ZvcihhPTA7YTx0OysrYSl0aGlzLnNldENlbGwoZSthLHIpfWVsc2UgZm9yKGE9ZTthPHRoaXMubGVuZ3RoOysrYSl0aGlzLnNldENlbGwoYSxyKTsyPT09dGhpcy5nZXRXaWR0aCh0aGlzLmxlbmd0aC0xKSYmdGhpcy5zZXRDZWxsRnJvbUNvZGVQb2ludCh0aGlzLmxlbmd0aC0xLDAsMSwobnVsbD09aT92b2lkIDA6aS5mZyl8fDAsKG51bGw9PWk/dm9pZCAwOmkuYmcpfHwwLChudWxsPT1pP3ZvaWQgMDppLmV4dGVuZGVkKXx8bmV3IHMuRXh0ZW5kZWRBdHRycyl9LGUucHJvdG90eXBlLmRlbGV0ZUNlbGxzPWZ1bmN0aW9uKGUsdCxyLGkpe2lmKGUlPXRoaXMubGVuZ3RoLHQ8dGhpcy5sZW5ndGgtZSl7Zm9yKHZhciBuPW5ldyBvLkNlbGxEYXRhLGE9MDthPHRoaXMubGVuZ3RoLWUtdDsrK2EpdGhpcy5zZXRDZWxsKGUrYSx0aGlzLmxvYWRDZWxsKGUrdCthLG4pKTtmb3IoYT10aGlzLmxlbmd0aC10O2E8dGhpcy5sZW5ndGg7KythKXRoaXMuc2V0Q2VsbChhLHIpfWVsc2UgZm9yKGE9ZTthPHRoaXMubGVuZ3RoOysrYSl0aGlzLnNldENlbGwoYSxyKTtlJiYyPT09dGhpcy5nZXRXaWR0aChlLTEpJiZ0aGlzLnNldENlbGxGcm9tQ29kZVBvaW50KGUtMSwwLDEsKG51bGw9PWk/dm9pZCAwOmkuZmcpfHwwLChudWxsPT1pP3ZvaWQgMDppLmJnKXx8MCwobnVsbD09aT92b2lkIDA6aS5leHRlbmRlZCl8fG5ldyBzLkV4dGVuZGVkQXR0cnMpLDAhPT10aGlzLmdldFdpZHRoKGUpfHx0aGlzLmhhc0NvbnRlbnQoZSl8fHRoaXMuc2V0Q2VsbEZyb21Db2RlUG9pbnQoZSwwLDEsKG51bGw9PWk/dm9pZCAwOmkuZmcpfHwwLChudWxsPT1pP3ZvaWQgMDppLmJnKXx8MCwobnVsbD09aT92b2lkIDA6aS5leHRlbmRlZCl8fG5ldyBzLkV4dGVuZGVkQXR0cnMpfSxlLnByb3RvdHlwZS5yZXBsYWNlQ2VsbHM9ZnVuY3Rpb24oZSx0LHIsaSl7Zm9yKGUmJjI9PT10aGlzLmdldFdpZHRoKGUtMSkmJnRoaXMuc2V0Q2VsbEZyb21Db2RlUG9pbnQoZS0xLDAsMSwobnVsbD09aT92b2lkIDA6aS5mZyl8fDAsKG51bGw9PWk/dm9pZCAwOmkuYmcpfHwwLChudWxsPT1pP3ZvaWQgMDppLmV4dGVuZGVkKXx8bmV3IHMuRXh0ZW5kZWRBdHRycyksdDx0aGlzLmxlbmd0aCYmMj09PXRoaXMuZ2V0V2lkdGgodC0xKSYmdGhpcy5zZXRDZWxsRnJvbUNvZGVQb2ludCh0LDAsMSwobnVsbD09aT92b2lkIDA6aS5mZyl8fDAsKG51bGw9PWk/dm9pZCAwOmkuYmcpfHwwLChudWxsPT1pP3ZvaWQgMDppLmV4dGVuZGVkKXx8bmV3IHMuRXh0ZW5kZWRBdHRycyk7ZTx0JiZlPHRoaXMubGVuZ3RoOyl0aGlzLnNldENlbGwoZSsrLHIpfSxlLnByb3RvdHlwZS5yZXNpemU9ZnVuY3Rpb24oZSx0KXtpZihlIT09dGhpcy5sZW5ndGgpe2lmKGU+dGhpcy5sZW5ndGgpe3ZhciByPW5ldyBVaW50MzJBcnJheSgzKmUpO3RoaXMubGVuZ3RoJiYoMyplPHRoaXMuX2RhdGEubGVuZ3RoP3Iuc2V0KHRoaXMuX2RhdGEuc3ViYXJyYXkoMCwzKmUpKTpyLnNldCh0aGlzLl9kYXRhKSksdGhpcy5fZGF0YT1yO2Zvcih2YXIgaT10aGlzLmxlbmd0aDtpPGU7KytpKXRoaXMuc2V0Q2VsbChpLHQpfWVsc2UgaWYoZSl7KHI9bmV3IFVpbnQzMkFycmF5KDMqZSkpLnNldCh0aGlzLl9kYXRhLnN1YmFycmF5KDAsMyplKSksdGhpcy5fZGF0YT1yO3ZhciBuPU9iamVjdC5rZXlzKHRoaXMuX2NvbWJpbmVkKTtmb3IoaT0wO2k8bi5sZW5ndGg7aSsrKXt2YXIgbz1wYXJzZUludChuW2ldLDEwKTtvPj1lJiZkZWxldGUgdGhpcy5fY29tYmluZWRbb119fWVsc2UgdGhpcy5fZGF0YT1uZXcgVWludDMyQXJyYXkoMCksdGhpcy5fY29tYmluZWQ9e307dGhpcy5sZW5ndGg9ZX19LGUucHJvdG90eXBlLmZpbGw9ZnVuY3Rpb24oZSl7dGhpcy5fY29tYmluZWQ9e30sdGhpcy5fZXh0ZW5kZWRBdHRycz17fTtmb3IodmFyIHQ9MDt0PHRoaXMubGVuZ3RoOysrdCl0aGlzLnNldENlbGwodCxlKX0sZS5wcm90b3R5cGUuY29weUZyb209ZnVuY3Rpb24oZSl7Zm9yKHZhciB0IGluIHRoaXMubGVuZ3RoIT09ZS5sZW5ndGg/dGhpcy5fZGF0YT1uZXcgVWludDMyQXJyYXkoZS5fZGF0YSk6dGhpcy5fZGF0YS5zZXQoZS5fZGF0YSksdGhpcy5sZW5ndGg9ZS5sZW5ndGgsdGhpcy5fY29tYmluZWQ9e30sZS5fY29tYmluZWQpdGhpcy5fY29tYmluZWRbdF09ZS5fY29tYmluZWRbdF07Zm9yKHZhciB0IGluIHRoaXMuX2V4dGVuZGVkQXR0cnM9e30sZS5fZXh0ZW5kZWRBdHRycyl0aGlzLl9leHRlbmRlZEF0dHJzW3RdPWUuX2V4dGVuZGVkQXR0cnNbdF07dGhpcy5pc1dyYXBwZWQ9ZS5pc1dyYXBwZWR9LGUucHJvdG90eXBlLmNsb25lPWZ1bmN0aW9uKCl7dmFyIHQ9bmV3IGUoMCk7Zm9yKHZhciByIGluIHQuX2RhdGE9bmV3IFVpbnQzMkFycmF5KHRoaXMuX2RhdGEpLHQubGVuZ3RoPXRoaXMubGVuZ3RoLHRoaXMuX2NvbWJpbmVkKXQuX2NvbWJpbmVkW3JdPXRoaXMuX2NvbWJpbmVkW3JdO2Zvcih2YXIgciBpbiB0aGlzLl9leHRlbmRlZEF0dHJzKXQuX2V4dGVuZGVkQXR0cnNbcl09dGhpcy5fZXh0ZW5kZWRBdHRyc1tyXTtyZXR1cm4gdC5pc1dyYXBwZWQ9dGhpcy5pc1dyYXBwZWQsdH0sZS5wcm90b3R5cGUuZ2V0VHJpbW1lZExlbmd0aD1mdW5jdGlvbigpe2Zvcih2YXIgZT10aGlzLmxlbmd0aC0xO2U+PTA7LS1lKWlmKDQxOTQzMDMmdGhpcy5fZGF0YVszKmUrMF0pcmV0dXJuIGUrKHRoaXMuX2RhdGFbMyplKzBdPj4yMik7cmV0dXJuIDB9LGUucHJvdG90eXBlLmNvcHlDZWxsc0Zyb209ZnVuY3Rpb24oZSx0LHIsaSxuKXt2YXIgbz1lLl9kYXRhO2lmKG4pZm9yKHZhciBzPWktMTtzPj0wO3MtLSlmb3IodmFyIGE9MDthPDM7YSsrKXRoaXMuX2RhdGFbMyoocitzKSthXT1vWzMqKHQrcykrYV07ZWxzZSBmb3Iocz0wO3M8aTtzKyspZm9yKGE9MDthPDM7YSsrKXRoaXMuX2RhdGFbMyoocitzKSthXT1vWzMqKHQrcykrYV07dmFyIGM9T2JqZWN0LmtleXMoZS5fY29tYmluZWQpO2ZvcihhPTA7YTxjLmxlbmd0aDthKyspe3ZhciBsPXBhcnNlSW50KGNbYV0sMTApO2w+PXQmJih0aGlzLl9jb21iaW5lZFtsLXQrcl09ZS5fY29tYmluZWRbbF0pfX0sZS5wcm90b3R5cGUudHJhbnNsYXRlVG9TdHJpbmc9ZnVuY3Rpb24oZSx0LHIpe3ZvaWQgMD09PWUmJihlPSExKSx2b2lkIDA9PT10JiYodD0wKSx2b2lkIDA9PT1yJiYocj10aGlzLmxlbmd0aCksZSYmKHI9TWF0aC5taW4ocix0aGlzLmdldFRyaW1tZWRMZW5ndGgoKSkpO2Zvcih2YXIgbz0iIjt0PHI7KXt2YXIgcz10aGlzLl9kYXRhWzMqdCswXSxhPTIwOTcxNTEmcztvKz0yMDk3MTUyJnM/dGhpcy5fY29tYmluZWRbdF06YT8oMCxpLnN0cmluZ0Zyb21Db2RlUG9pbnQpKGEpOm4uV0hJVEVTUEFDRV9DRUxMX0NIQVIsdCs9cz4+MjJ8fDF9cmV0dXJuIG99LGV9KCk7dC5CdWZmZXJMaW5lPWF9LDQ4NDE6KGUsdCk9PntPYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC5nZXRSYW5nZUxlbmd0aD12b2lkIDAsdC5nZXRSYW5nZUxlbmd0aD1mdW5jdGlvbihlLHQpe2lmKGUuc3RhcnQueT5lLmVuZC55KXRocm93IG5ldyBFcnJvcigiQnVmZmVyIHJhbmdlIGVuZCAoIitlLmVuZC54KyIsICIrZS5lbmQueSsiKSBjYW5ub3QgYmUgYmVmb3JlIHN0YXJ0ICgiK2Uuc3RhcnQueCsiLCAiK2Uuc3RhcnQueSsiKSIpO3JldHVybiB0KihlLmVuZC55LWUuc3RhcnQueSkrKGUuZW5kLngtZS5zdGFydC54KzEpfX0sNDYzNDooZSx0KT0+e2Z1bmN0aW9uIHIoZSx0LHIpe2lmKHQ9PT1lLmxlbmd0aC0xKXJldHVybiBlW3RdLmdldFRyaW1tZWRMZW5ndGgoKTt2YXIgaT0hZVt0XS5oYXNDb250ZW50KHItMSkmJjE9PT1lW3RdLmdldFdpZHRoKHItMSksbj0yPT09ZVt0KzFdLmdldFdpZHRoKDApO3JldHVybiBpJiZuP3ItMTpyfU9iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LCJfX2VzTW9kdWxlIix7dmFsdWU6ITB9KSx0LmdldFdyYXBwZWRMaW5lVHJpbW1lZExlbmd0aD10LnJlZmxvd1NtYWxsZXJHZXROZXdMaW5lTGVuZ3Rocz10LnJlZmxvd0xhcmdlckFwcGx5TmV3TGF5b3V0PXQucmVmbG93TGFyZ2VyQ3JlYXRlTmV3TGF5b3V0PXQucmVmbG93TGFyZ2VyR2V0TGluZXNUb1JlbW92ZT12b2lkIDAsdC5yZWZsb3dMYXJnZXJHZXRMaW5lc1RvUmVtb3ZlPWZ1bmN0aW9uKGUsdCxpLG4sbyl7Zm9yKHZhciBzPVtdLGE9MDthPGUubGVuZ3RoLTE7YSsrKXt2YXIgYz1hLGw9ZS5nZXQoKytjKTtpZihsLmlzV3JhcHBlZCl7Zm9yKHZhciB1PVtlLmdldChhKV07YzxlLmxlbmd0aCYmbC5pc1dyYXBwZWQ7KXUucHVzaChsKSxsPWUuZ2V0KCsrYyk7aWYobj49YSYmbjxjKWErPXUubGVuZ3RoLTE7ZWxzZXtmb3IodmFyIGg9MCxmPXIodSxoLHQpLF89MSxkPTA7Xzx1Lmxlbmd0aDspe3ZhciBwPXIodSxfLHQpLHY9cC1kLGc9aS1mLHk9TWF0aC5taW4odixnKTt1W2hdLmNvcHlDZWxsc0Zyb20odVtfXSxkLGYseSwhMSksKGYrPXkpPT09aSYmKGgrKyxmPTApLChkKz15KT09PXAmJihfKyssZD0wKSwwPT09ZiYmMCE9PWgmJjI9PT11W2gtMV0uZ2V0V2lkdGgoaS0xKSYmKHVbaF0uY29weUNlbGxzRnJvbSh1W2gtMV0saS0xLGYrKywxLCExKSx1W2gtMV0uc2V0Q2VsbChpLTEsbykpfXVbaF0ucmVwbGFjZUNlbGxzKGYsaSxvKTtmb3IodmFyIG09MCxiPXUubGVuZ3RoLTE7Yj4wJiYoYj5ofHwwPT09dVtiXS5nZXRUcmltbWVkTGVuZ3RoKCkpO2ItLSltKys7bT4wJiYocy5wdXNoKGErdS5sZW5ndGgtbSkscy5wdXNoKG0pKSxhKz11Lmxlbmd0aC0xfX19cmV0dXJuIHN9LHQucmVmbG93TGFyZ2VyQ3JlYXRlTmV3TGF5b3V0PWZ1bmN0aW9uKGUsdCl7Zm9yKHZhciByPVtdLGk9MCxuPXRbaV0sbz0wLHM9MDtzPGUubGVuZ3RoO3MrKylpZihuPT09cyl7dmFyIGE9dFsrK2ldO2Uub25EZWxldGVFbWl0dGVyLmZpcmUoe2luZGV4OnMtbyxhbW91bnQ6YX0pLHMrPWEtMSxvKz1hLG49dFsrK2ldfWVsc2Ugci5wdXNoKHMpO3JldHVybntsYXlvdXQ6cixjb3VudFJlbW92ZWQ6b319LHQucmVmbG93TGFyZ2VyQXBwbHlOZXdMYXlvdXQ9ZnVuY3Rpb24oZSx0KXtmb3IodmFyIHI9W10saT0wO2k8dC5sZW5ndGg7aSsrKXIucHVzaChlLmdldCh0W2ldKSk7Zm9yKGk9MDtpPHIubGVuZ3RoO2krKyllLnNldChpLHJbaV0pO2UubGVuZ3RoPXQubGVuZ3RofSx0LnJlZmxvd1NtYWxsZXJHZXROZXdMaW5lTGVuZ3Rocz1mdW5jdGlvbihlLHQsaSl7Zm9yKHZhciBuPVtdLG89ZS5tYXAoKGZ1bmN0aW9uKGksbil7cmV0dXJuIHIoZSxuLHQpfSkpLnJlZHVjZSgoZnVuY3Rpb24oZSx0KXtyZXR1cm4gZSt0fSkpLHM9MCxhPTAsYz0wO2M8bzspe2lmKG8tYzxpKXtuLnB1c2goby1jKTticmVha31zKz1pO3ZhciBsPXIoZSxhLHQpO3M+bCYmKHMtPWwsYSsrKTt2YXIgdT0yPT09ZVthXS5nZXRXaWR0aChzLTEpO3UmJnMtLTt2YXIgaD11P2ktMTppO24ucHVzaChoKSxjKz1ofXJldHVybiBufSx0LmdldFdyYXBwZWRMaW5lVHJpbW1lZExlbmd0aD1yfSw1Mjk1OmZ1bmN0aW9uKGUsdCxyKXt2YXIgaSxuPXRoaXMmJnRoaXMuX19leHRlbmRzfHwoaT1mdW5jdGlvbihlLHQpe3JldHVybiBpPU9iamVjdC5zZXRQcm90b3R5cGVPZnx8e19fcHJvdG9fXzpbXX1pbnN0YW5jZW9mIEFycmF5JiZmdW5jdGlvbihlLHQpe2UuX19wcm90b19fPXR9fHxmdW5jdGlvbihlLHQpe2Zvcih2YXIgciBpbiB0KU9iamVjdC5wcm90b3R5cGUuaGFzT3duUHJvcGVydHkuY2FsbCh0LHIpJiYoZVtyXT10W3JdKX0saShlLHQpfSxmdW5jdGlvbihlLHQpe2lmKCJmdW5jdGlvbiIhPXR5cGVvZiB0JiZudWxsIT09dCl0aHJvdyBuZXcgVHlwZUVycm9yKCJDbGFzcyBleHRlbmRzIHZhbHVlICIrU3RyaW5nKHQpKyIgaXMgbm90IGEgY29uc3RydWN0b3Igb3IgbnVsbCIpO2Z1bmN0aW9uIHIoKXt0aGlzLmNvbnN0cnVjdG9yPWV9aShlLHQpLGUucHJvdG90eXBlPW51bGw9PT10P09iamVjdC5jcmVhdGUodCk6KHIucHJvdG90eXBlPXQucHJvdG90eXBlLG5ldyByKX0pO09iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LCJfX2VzTW9kdWxlIix7dmFsdWU6ITB9KSx0LkJ1ZmZlclNldD12b2lkIDA7dmFyIG89cig5MDkyKSxzPXIoODQ2MCksYT1mdW5jdGlvbihlKXtmdW5jdGlvbiB0KHQscil7dmFyIGk9ZS5jYWxsKHRoaXMpfHx0aGlzO3JldHVybiBpLl9vcHRpb25zU2VydmljZT10LGkuX2J1ZmZlclNlcnZpY2U9cixpLl9vbkJ1ZmZlckFjdGl2YXRlPWkucmVnaXN0ZXIobmV3IHMuRXZlbnRFbWl0dGVyKSxpLnJlc2V0KCksaX1yZXR1cm4gbih0LGUpLE9iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LnByb3RvdHlwZSwib25CdWZmZXJBY3RpdmF0ZSIse2dldDpmdW5jdGlvbigpe3JldHVybiB0aGlzLl9vbkJ1ZmZlckFjdGl2YXRlLmV2ZW50fSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLHQucHJvdG90eXBlLnJlc2V0PWZ1bmN0aW9uKCl7dGhpcy5fbm9ybWFsPW5ldyBvLkJ1ZmZlcighMCx0aGlzLl9vcHRpb25zU2VydmljZSx0aGlzLl9idWZmZXJTZXJ2aWNlKSx0aGlzLl9ub3JtYWwuZmlsbFZpZXdwb3J0Um93cygpLHRoaXMuX2FsdD1uZXcgby5CdWZmZXIoITEsdGhpcy5fb3B0aW9uc1NlcnZpY2UsdGhpcy5fYnVmZmVyU2VydmljZSksdGhpcy5fYWN0aXZlQnVmZmVyPXRoaXMuX25vcm1hbCx0aGlzLl9vbkJ1ZmZlckFjdGl2YXRlLmZpcmUoe2FjdGl2ZUJ1ZmZlcjp0aGlzLl9ub3JtYWwsaW5hY3RpdmVCdWZmZXI6dGhpcy5fYWx0fSksdGhpcy5zZXR1cFRhYlN0b3BzKCl9LE9iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LnByb3RvdHlwZSwiYWx0Iix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX2FsdH0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxPYmplY3QuZGVmaW5lUHJvcGVydHkodC5wcm90b3R5cGUsImFjdGl2ZSIse2dldDpmdW5jdGlvbigpe3JldHVybiB0aGlzLl9hY3RpdmVCdWZmZXJ9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KHQucHJvdG90eXBlLCJub3JtYWwiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fbm9ybWFsfSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLHQucHJvdG90eXBlLmFjdGl2YXRlTm9ybWFsQnVmZmVyPWZ1bmN0aW9uKCl7dGhpcy5fYWN0aXZlQnVmZmVyIT09dGhpcy5fbm9ybWFsJiYodGhpcy5fbm9ybWFsLng9dGhpcy5fYWx0LngsdGhpcy5fbm9ybWFsLnk9dGhpcy5fYWx0LnksdGhpcy5fYWx0LmNsZWFyKCksdGhpcy5fYWN0aXZlQnVmZmVyPXRoaXMuX25vcm1hbCx0aGlzLl9vbkJ1ZmZlckFjdGl2YXRlLmZpcmUoe2FjdGl2ZUJ1ZmZlcjp0aGlzLl9ub3JtYWwsaW5hY3RpdmVCdWZmZXI6dGhpcy5fYWx0fSkpfSx0LnByb3RvdHlwZS5hY3RpdmF0ZUFsdEJ1ZmZlcj1mdW5jdGlvbihlKXt0aGlzLl9hY3RpdmVCdWZmZXIhPT10aGlzLl9hbHQmJih0aGlzLl9hbHQuZmlsbFZpZXdwb3J0Um93cyhlKSx0aGlzLl9hbHQueD10aGlzLl9ub3JtYWwueCx0aGlzLl9hbHQueT10aGlzLl9ub3JtYWwueSx0aGlzLl9hY3RpdmVCdWZmZXI9dGhpcy5fYWx0LHRoaXMuX29uQnVmZmVyQWN0aXZhdGUuZmlyZSh7YWN0aXZlQnVmZmVyOnRoaXMuX2FsdCxpbmFjdGl2ZUJ1ZmZlcjp0aGlzLl9ub3JtYWx9KSl9LHQucHJvdG90eXBlLnJlc2l6ZT1mdW5jdGlvbihlLHQpe3RoaXMuX25vcm1hbC5yZXNpemUoZSx0KSx0aGlzLl9hbHQucmVzaXplKGUsdCl9LHQucHJvdG90eXBlLnNldHVwVGFiU3RvcHM9ZnVuY3Rpb24oZSl7dGhpcy5fbm9ybWFsLnNldHVwVGFiU3RvcHMoZSksdGhpcy5fYWx0LnNldHVwVGFiU3RvcHMoZSl9LHR9KHIoODQ0KS5EaXNwb3NhYmxlKTt0LkJ1ZmZlclNldD1hfSw1MTE6ZnVuY3Rpb24oZSx0LHIpe3ZhciBpLG49dGhpcyYmdGhpcy5fX2V4dGVuZHN8fChpPWZ1bmN0aW9uKGUsdCl7cmV0dXJuIGk9T2JqZWN0LnNldFByb3RvdHlwZU9mfHx7X19wcm90b19fOltdfWluc3RhbmNlb2YgQXJyYXkmJmZ1bmN0aW9uKGUsdCl7ZS5fX3Byb3RvX189dH18fGZ1bmN0aW9uKGUsdCl7Zm9yKHZhciByIGluIHQpT2JqZWN0LnByb3RvdHlwZS5oYXNPd25Qcm9wZXJ0eS5jYWxsKHQscikmJihlW3JdPXRbcl0pfSxpKGUsdCl9LGZ1bmN0aW9uKGUsdCl7aWYoImZ1bmN0aW9uIiE9dHlwZW9mIHQmJm51bGwhPT10KXRocm93IG5ldyBUeXBlRXJyb3IoIkNsYXNzIGV4dGVuZHMgdmFsdWUgIitTdHJpbmcodCkrIiBpcyBub3QgYSBjb25zdHJ1Y3RvciBvciBudWxsIik7ZnVuY3Rpb24gcigpe3RoaXMuY29uc3RydWN0b3I9ZX1pKGUsdCksZS5wcm90b3R5cGU9bnVsbD09PXQ/T2JqZWN0LmNyZWF0ZSh0KTooci5wcm90b3R5cGU9dC5wcm90b3R5cGUsbmV3IHIpfSk7T2JqZWN0LmRlZmluZVByb3BlcnR5KHQsIl9fZXNNb2R1bGUiLHt2YWx1ZTohMH0pLHQuQ2VsbERhdGE9dm9pZCAwO3ZhciBvPXIoNDgyKSxzPXIoNjQzKSxhPXIoMzczNCksYz1mdW5jdGlvbihlKXtmdW5jdGlvbiB0KCl7dmFyIHQ9bnVsbCE9PWUmJmUuYXBwbHkodGhpcyxhcmd1bWVudHMpfHx0aGlzO3JldHVybiB0LmNvbnRlbnQ9MCx0LmZnPTAsdC5iZz0wLHQuZXh0ZW5kZWQ9bmV3IGEuRXh0ZW5kZWRBdHRycyx0LmNvbWJpbmVkRGF0YT0iIix0fXJldHVybiBuKHQsZSksdC5mcm9tQ2hhckRhdGE9ZnVuY3Rpb24oZSl7dmFyIHI9bmV3IHQ7cmV0dXJuIHIuc2V0RnJvbUNoYXJEYXRhKGUpLHJ9LHQucHJvdG90eXBlLmlzQ29tYmluZWQ9ZnVuY3Rpb24oKXtyZXR1cm4gMjA5NzE1MiZ0aGlzLmNvbnRlbnR9LHQucHJvdG90eXBlLmdldFdpZHRoPWZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuY29udGVudD4+MjJ9LHQucHJvdG90eXBlLmdldENoYXJzPWZ1bmN0aW9uKCl7cmV0dXJuIDIwOTcxNTImdGhpcy5jb250ZW50P3RoaXMuY29tYmluZWREYXRhOjIwOTcxNTEmdGhpcy5jb250ZW50PygwLG8uc3RyaW5nRnJvbUNvZGVQb2ludCkoMjA5NzE1MSZ0aGlzLmNvbnRlbnQpOiIifSx0LnByb3RvdHlwZS5nZXRDb2RlPWZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuaXNDb21iaW5lZCgpP3RoaXMuY29tYmluZWREYXRhLmNoYXJDb2RlQXQodGhpcy5jb21iaW5lZERhdGEubGVuZ3RoLTEpOjIwOTcxNTEmdGhpcy5jb250ZW50fSx0LnByb3RvdHlwZS5zZXRGcm9tQ2hhckRhdGE9ZnVuY3Rpb24oZSl7dGhpcy5mZz1lW3MuQ0hBUl9EQVRBX0FUVFJfSU5ERVhdLHRoaXMuYmc9MDt2YXIgdD0hMTtpZihlW3MuQ0hBUl9EQVRBX0NIQVJfSU5ERVhdLmxlbmd0aD4yKXQ9ITA7ZWxzZSBpZigyPT09ZVtzLkNIQVJfREFUQV9DSEFSX0lOREVYXS5sZW5ndGgpe3ZhciByPWVbcy5DSEFSX0RBVEFfQ0hBUl9JTkRFWF0uY2hhckNvZGVBdCgwKTtpZig1NTI5Njw9ciYmcjw9NTYzMTkpe3ZhciBpPWVbcy5DSEFSX0RBVEFfQ0hBUl9JTkRFWF0uY2hhckNvZGVBdCgxKTs1NjMyMDw9aSYmaTw9NTczNDM/dGhpcy5jb250ZW50PTEwMjQqKHItNTUyOTYpK2ktNTYzMjArNjU1MzZ8ZVtzLkNIQVJfREFUQV9XSURUSF9JTkRFWF08PDIyOnQ9ITB9ZWxzZSB0PSEwfWVsc2UgdGhpcy5jb250ZW50PWVbcy5DSEFSX0RBVEFfQ0hBUl9JTkRFWF0uY2hhckNvZGVBdCgwKXxlW3MuQ0hBUl9EQVRBX1dJRFRIX0lOREVYXTw8MjI7dCYmKHRoaXMuY29tYmluZWREYXRhPWVbcy5DSEFSX0RBVEFfQ0hBUl9JTkRFWF0sdGhpcy5jb250ZW50PTIwOTcxNTJ8ZVtzLkNIQVJfREFUQV9XSURUSF9JTkRFWF08PDIyKX0sdC5wcm90b3R5cGUuZ2V0QXNDaGFyRGF0YT1mdW5jdGlvbigpe3JldHVyblt0aGlzLmZnLHRoaXMuZ2V0Q2hhcnMoKSx0aGlzLmdldFdpZHRoKCksdGhpcy5nZXRDb2RlKCldfSx0fShhLkF0dHJpYnV0ZURhdGEpO3QuQ2VsbERhdGE9Y30sNjQzOihlLHQpPT57T2JqZWN0LmRlZmluZVByb3BlcnR5KHQsIl9fZXNNb2R1bGUiLHt2YWx1ZTohMH0pLHQuV0hJVEVTUEFDRV9DRUxMX0NPREU9dC5XSElURVNQQUNFX0NFTExfV0lEVEg9dC5XSElURVNQQUNFX0NFTExfQ0hBUj10Lk5VTExfQ0VMTF9DT0RFPXQuTlVMTF9DRUxMX1dJRFRIPXQuTlVMTF9DRUxMX0NIQVI9dC5DSEFSX0RBVEFfQ09ERV9JTkRFWD10LkNIQVJfREFUQV9XSURUSF9JTkRFWD10LkNIQVJfREFUQV9DSEFSX0lOREVYPXQuQ0hBUl9EQVRBX0FUVFJfSU5ERVg9dC5ERUZBVUxUX0FUVFI9dC5ERUZBVUxUX0NPTE9SPXZvaWQgMCx0LkRFRkFVTFRfQ09MT1I9MjU2LHQuREVGQVVMVF9BVFRSPTI1Nnx0LkRFRkFVTFRfQ09MT1I8PDksdC5DSEFSX0RBVEFfQVRUUl9JTkRFWD0wLHQuQ0hBUl9EQVRBX0NIQVJfSU5ERVg9MSx0LkNIQVJfREFUQV9XSURUSF9JTkRFWD0yLHQuQ0hBUl9EQVRBX0NPREVfSU5ERVg9Myx0Lk5VTExfQ0VMTF9DSEFSPSIiLHQuTlVMTF9DRUxMX1dJRFRIPTEsdC5OVUxMX0NFTExfQ09ERT0wLHQuV0hJVEVTUEFDRV9DRUxMX0NIQVI9IiAiLHQuV0hJVEVTUEFDRV9DRUxMX1dJRFRIPTEsdC5XSElURVNQQUNFX0NFTExfQ09ERT0zMn0sNDg2MzpmdW5jdGlvbihlLHQscil7dmFyIGksbj10aGlzJiZ0aGlzLl9fZXh0ZW5kc3x8KGk9ZnVuY3Rpb24oZSx0KXtyZXR1cm4gaT1PYmplY3Quc2V0UHJvdG90eXBlT2Z8fHtfX3Byb3RvX186W119aW5zdGFuY2VvZiBBcnJheSYmZnVuY3Rpb24oZSx0KXtlLl9fcHJvdG9fXz10fXx8ZnVuY3Rpb24oZSx0KXtmb3IodmFyIHIgaW4gdClPYmplY3QucHJvdG90eXBlLmhhc093blByb3BlcnR5LmNhbGwodCxyKSYmKGVbcl09dFtyXSl9LGkoZSx0KX0sZnVuY3Rpb24oZSx0KXtpZigiZnVuY3Rpb24iIT10eXBlb2YgdCYmbnVsbCE9PXQpdGhyb3cgbmV3IFR5cGVFcnJvcigiQ2xhc3MgZXh0ZW5kcyB2YWx1ZSAiK1N0cmluZyh0KSsiIGlzIG5vdCBhIGNvbnN0cnVjdG9yIG9yIG51bGwiKTtmdW5jdGlvbiByKCl7dGhpcy5jb25zdHJ1Y3Rvcj1lfWkoZSx0KSxlLnByb3RvdHlwZT1udWxsPT09dD9PYmplY3QuY3JlYXRlKHQpOihyLnByb3RvdHlwZT10LnByb3RvdHlwZSxuZXcgcil9KTtPYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC5NYXJrZXI9dm9pZCAwO3ZhciBvPXIoODQ2MCkscz1mdW5jdGlvbihlKXtmdW5jdGlvbiB0KHIpe3ZhciBpPWUuY2FsbCh0aGlzKXx8dGhpcztyZXR1cm4gaS5saW5lPXIsaS5faWQ9dC5fbmV4dElkKyssaS5pc0Rpc3Bvc2VkPSExLGkuX29uRGlzcG9zZT1uZXcgby5FdmVudEVtaXR0ZXIsaX1yZXR1cm4gbih0LGUpLE9iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LnByb3RvdHlwZSwiaWQiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5faWR9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KHQucHJvdG90eXBlLCJvbkRpc3Bvc2UiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fb25EaXNwb3NlLmV2ZW50fSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLHQucHJvdG90eXBlLmRpc3Bvc2U9ZnVuY3Rpb24oKXt0aGlzLmlzRGlzcG9zZWR8fCh0aGlzLmlzRGlzcG9zZWQ9ITAsdGhpcy5saW5lPS0xLHRoaXMuX29uRGlzcG9zZS5maXJlKCksZS5wcm90b3R5cGUuZGlzcG9zZS5jYWxsKHRoaXMpKX0sdC5fbmV4dElkPTEsdH0ocig4NDQpLkRpc3Bvc2FibGUpO3QuTWFya2VyPXN9LDcxMTY6KGUsdCk9PntPYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC5ERUZBVUxUX0NIQVJTRVQ9dC5DSEFSU0VUUz12b2lkIDAsdC5DSEFSU0VUUz17fSx0LkRFRkFVTFRfQ0hBUlNFVD10LkNIQVJTRVRTLkIsdC5DSEFSU0VUU1swXT17ImAiOiLil4YiLGE6IuKWkiIsYjoi4pCJIixjOiLikIwiLGQ6IuKQjSIsZToi4pCKIixmOiLCsCIsZzoiwrEiLGg6IuKQpCIsaToi4pCLIixqOiLilJgiLGs6IuKUkCIsbDoi4pSMIixtOiLilJQiLG46IuKUvCIsbzoi4o66IixwOiLijrsiLHE6IuKUgCIscjoi4o68IixzOiLijr0iLHQ6IuKUnCIsdToi4pSkIix2OiLilLQiLHc6IuKUrCIseDoi4pSCIix5OiLiiaQiLHo6IuKJpSIsInsiOiLPgCIsInwiOiLiiaAiLCJ9IjoiwqMiLCJ+IjoiwrcifSx0LkNIQVJTRVRTLkE9eyIjIjoiwqMifSx0LkNIQVJTRVRTLkI9dm9pZCAwLHQuQ0hBUlNFVFNbNF09eyIjIjoiwqMiLCJAIjoiwr4iLCJbIjoiaWoiLCJcXCI6IsK9IiwiXSI6InwiLCJ7IjoiwqgiLCJ8IjoiZiIsIn0iOiLCvCIsIn4iOiLCtCJ9LHQuQ0hBUlNFVFMuQz10LkNIQVJTRVRTWzVdPXsiWyI6IsOEIiwiXFwiOiLDliIsIl0iOiLDhSIsIl4iOiLDnCIsImAiOiLDqSIsInsiOiLDpCIsInwiOiLDtiIsIn0iOiLDpSIsIn4iOiLDvCJ9LHQuQ0hBUlNFVFMuUj17IiMiOiLCoyIsIkAiOiLDoCIsIlsiOiLCsCIsIlxcIjoiw6ciLCJdIjoiwqciLCJ7Ijoiw6kiLCJ8Ijoiw7kiLCJ9Ijoiw6giLCJ+IjoiwqgifSx0LkNIQVJTRVRTLlE9eyJAIjoiw6AiLCJbIjoiw6IiLCJcXCI6IsOnIiwiXSI6IsOqIiwiXiI6IsOuIiwiYCI6IsO0IiwieyI6IsOpIiwifCI6IsO5IiwifSI6IsOoIiwifiI6IsO7In0sdC5DSEFSU0VUUy5LPXsiQCI6IsKnIiwiWyI6IsOEIiwiXFwiOiLDliIsIl0iOiLDnCIsInsiOiLDpCIsInwiOiLDtiIsIn0iOiLDvCIsIn4iOiLDnyJ9LHQuQ0hBUlNFVFMuWT17IiMiOiLCoyIsIkAiOiLCpyIsIlsiOiLCsCIsIlxcIjoiw6ciLCJdIjoiw6kiLCJgIjoiw7kiLCJ7Ijoiw6AiLCJ8Ijoiw7IiLCJ9Ijoiw6giLCJ+Ijoiw6wifSx0LkNIQVJTRVRTLkU9dC5DSEFSU0VUU1s2XT17IkAiOiLDhCIsIlsiOiLDhiIsIlxcIjoiw5giLCJdIjoiw4UiLCJeIjoiw5wiLCJgIjoiw6QiLCJ7Ijoiw6YiLCJ8Ijoiw7giLCJ9Ijoiw6UiLCJ+Ijoiw7wifSx0LkNIQVJTRVRTLlo9eyIjIjoiwqMiLCJAIjoiwqciLCJbIjoiwqEiLCJcXCI6IsORIiwiXSI6IsK/IiwieyI6IsKwIiwifCI6IsOxIiwifSI6IsOnIn0sdC5DSEFSU0VUUy5IPXQuQ0hBUlNFVFNbN109eyJAIjoiw4kiLCJbIjoiw4QiLCJcXCI6IsOWIiwiXSI6IsOFIiwiXiI6IsOcIiwiYCI6IsOpIiwieyI6IsOkIiwifCI6IsO2IiwifSI6IsOlIiwifiI6IsO8In0sdC5DSEFSU0VUU1siPSJdPXsiIyI6IsO5IiwiQCI6IsOgIiwiWyI6IsOpIiwiXFwiOiLDpyIsIl0iOiLDqiIsIl4iOiLDriIsXzoiw6giLCJgIjoiw7QiLCJ7Ijoiw6QiLCJ8Ijoiw7YiLCJ9Ijoiw7wiLCJ+Ijoiw7sifX0sMjU4NDooZSx0KT0+e3ZhciByLGk7T2JqZWN0LmRlZmluZVByb3BlcnR5KHQsIl9fZXNNb2R1bGUiLHt2YWx1ZTohMH0pLHQuQzE9dC5DMD12b2lkIDAsKGk9dC5DMHx8KHQuQzA9e30pKS5OVUw9IlwwIixpLlNPSD0iASIsaS5TVFg9IgIiLGkuRVRYPSIDIixpLkVPVD0iBCIsaS5FTlE9IgUiLGkuQUNLPSIGIixpLkJFTD0iByIsaS5CUz0iXGIiLGkuSFQ9Ilx0IixpLkxGPSJcbiIsaS5WVD0iXHYiLGkuRkY9IlxmIixpLkNSPSJcciIsaS5TTz0iDiIsaS5TST0iDyIsaS5ETEU9IhAiLGkuREMxPSIRIixpLkRDMj0iEiIsaS5EQzM9IhMiLGkuREM0PSIUIixpLk5BSz0iFSIsaS5TWU49IhYiLGkuRVRCPSIXIixpLkNBTj0iGCIsaS5FTT0iGSIsaS5TVUI9IhoiLGkuRVNDPSIbIixpLkZTPSIcIixpLkdTPSIdIixpLlJTPSIeIixpLlVTPSIfIixpLlNQPSIgIixpLkRFTD0ifyIsKHI9dC5DMXx8KHQuQzE9e30pKS5QQUQ9IsKAIixyLkhPUD0iwoEiLHIuQlBIPSLCgiIsci5OQkg9IsKDIixyLklORD0iwoQiLHIuTkVMPSLChSIsci5TU0E9IsKGIixyLkVTQT0iwociLHIuSFRTPSLCiCIsci5IVEo9IsKJIixyLlZUUz0iwooiLHIuUExEPSLCiyIsci5QTFU9IsKMIixyLlJJPSLCjSIsci5TUzI9IsKOIixyLlNTMz0iwo8iLHIuRENTPSLCkCIsci5QVTE9IsKRIixyLlBVMj0iwpIiLHIuU1RTPSLCkyIsci5DQ0g9IsKUIixyLk1XPSLClSIsci5TUEE9IsKWIixyLkVQQT0iwpciLHIuU09TPSLCmCIsci5TR0NJPSLCmSIsci5TQ0k9IsKaIixyLkNTST0iwpsiLHIuU1Q9IsKcIixyLk9TQz0iwp0iLHIuUE09IsKeIixyLkFQQz0iwp8ifSw3Mzk5OihlLHQscik9PntPYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC5ldmFsdWF0ZUtleWJvYXJkRXZlbnQ9dm9pZCAwO3ZhciBpPXIoMjU4NCksbj17NDg6WyIwIiwiKSJdLDQ5OlsiMSIsIiEiXSw1MDpbIjIiLCJAIl0sNTE6WyIzIiwiIyJdLDUyOlsiNCIsIiQiXSw1MzpbIjUiLCIlIl0sNTQ6WyI2IiwiXiJdLDU1OlsiNyIsIiYiXSw1NjpbIjgiLCIqIl0sNTc6WyI5IiwiKCJdLDE4NjpbIjsiLCI6Il0sMTg3OlsiPSIsIisiXSwxODg6WyIsIiwiPCJdLDE4OTpbIi0iLCJfIl0sMTkwOlsiLiIsIj4iXSwxOTE6WyIvIiwiPyJdLDE5MjpbImAiLCJ+Il0sMjE5OlsiWyIsInsiXSwyMjA6WyJcXCIsInwiXSwyMjE6WyJdIiwifSJdLDIyMjpbIiciLCciJ119O3QuZXZhbHVhdGVLZXlib2FyZEV2ZW50PWZ1bmN0aW9uKGUsdCxyLG8pe3ZhciBzPXt0eXBlOjAsY2FuY2VsOiExLGtleTp2b2lkIDB9LGE9KGUuc2hpZnRLZXk/MTowKXwoZS5hbHRLZXk/MjowKXwoZS5jdHJsS2V5PzQ6MCl8KGUubWV0YUtleT84OjApO3N3aXRjaChlLmtleUNvZGUpe2Nhc2UgMDoiVUlLZXlJbnB1dFVwQXJyb3ciPT09ZS5rZXk/cy5rZXk9dD9pLkMwLkVTQysiT0EiOmkuQzAuRVNDKyJbQSI6IlVJS2V5SW5wdXRMZWZ0QXJyb3ciPT09ZS5rZXk/cy5rZXk9dD9pLkMwLkVTQysiT0QiOmkuQzAuRVNDKyJbRCI6IlVJS2V5SW5wdXRSaWdodEFycm93Ij09PWUua2V5P3Mua2V5PXQ/aS5DMC5FU0MrIk9DIjppLkMwLkVTQysiW0MiOiJVSUtleUlucHV0RG93bkFycm93Ij09PWUua2V5JiYocy5rZXk9dD9pLkMwLkVTQysiT0IiOmkuQzAuRVNDKyJbQiIpO2JyZWFrO2Nhc2UgODppZihlLnNoaWZ0S2V5KXtzLmtleT1pLkMwLkJTO2JyZWFrfWlmKGUuYWx0S2V5KXtzLmtleT1pLkMwLkVTQytpLkMwLkRFTDticmVha31zLmtleT1pLkMwLkRFTDticmVhaztjYXNlIDk6aWYoZS5zaGlmdEtleSl7cy5rZXk9aS5DMC5FU0MrIltaIjticmVha31zLmtleT1pLkMwLkhULHMuY2FuY2VsPSEwO2JyZWFrO2Nhc2UgMTM6cy5rZXk9ZS5hbHRLZXk/aS5DMC5FU0MraS5DMC5DUjppLkMwLkNSLHMuY2FuY2VsPSEwO2JyZWFrO2Nhc2UgMjc6cy5rZXk9aS5DMC5FU0MsZS5hbHRLZXkmJihzLmtleT1pLkMwLkVTQytpLkMwLkVTQykscy5jYW5jZWw9ITA7YnJlYWs7Y2FzZSAzNzppZihlLm1ldGFLZXkpYnJlYWs7YT8ocy5rZXk9aS5DMC5FU0MrIlsxOyIrKGErMSkrIkQiLHMua2V5PT09aS5DMC5FU0MrIlsxOzNEIiYmKHMua2V5PWkuQzAuRVNDKyhyPyJiIjoiWzE7NUQiKSkpOnMua2V5PXQ/aS5DMC5FU0MrIk9EIjppLkMwLkVTQysiW0QiO2JyZWFrO2Nhc2UgMzk6aWYoZS5tZXRhS2V5KWJyZWFrO2E/KHMua2V5PWkuQzAuRVNDKyJbMTsiKyhhKzEpKyJDIixzLmtleT09PWkuQzAuRVNDKyJbMTszQyImJihzLmtleT1pLkMwLkVTQysocj8iZiI6IlsxOzVDIikpKTpzLmtleT10P2kuQzAuRVNDKyJPQyI6aS5DMC5FU0MrIltDIjticmVhaztjYXNlIDM4OmlmKGUubWV0YUtleSlicmVhazthPyhzLmtleT1pLkMwLkVTQysiWzE7IisoYSsxKSsiQSIscnx8cy5rZXkhPT1pLkMwLkVTQysiWzE7M0EifHwocy5rZXk9aS5DMC5FU0MrIlsxOzVBIikpOnMua2V5PXQ/aS5DMC5FU0MrIk9BIjppLkMwLkVTQysiW0EiO2JyZWFrO2Nhc2UgNDA6aWYoZS5tZXRhS2V5KWJyZWFrO2E/KHMua2V5PWkuQzAuRVNDKyJbMTsiKyhhKzEpKyJCIixyfHxzLmtleSE9PWkuQzAuRVNDKyJbMTszQiJ8fChzLmtleT1pLkMwLkVTQysiWzE7NUIiKSk6cy5rZXk9dD9pLkMwLkVTQysiT0IiOmkuQzAuRVNDKyJbQiI7YnJlYWs7Y2FzZSA0NTplLnNoaWZ0S2V5fHxlLmN0cmxLZXl8fChzLmtleT1pLkMwLkVTQysiWzJ+Iik7YnJlYWs7Y2FzZSA0NjpzLmtleT1hP2kuQzAuRVNDKyJbMzsiKyhhKzEpKyJ+IjppLkMwLkVTQysiWzN+IjticmVhaztjYXNlIDM2OnMua2V5PWE/aS5DMC5FU0MrIlsxOyIrKGErMSkrIkgiOnQ/aS5DMC5FU0MrIk9IIjppLkMwLkVTQysiW0giO2JyZWFrO2Nhc2UgMzU6cy5rZXk9YT9pLkMwLkVTQysiWzE7IisoYSsxKSsiRiI6dD9pLkMwLkVTQysiT0YiOmkuQzAuRVNDKyJbRiI7YnJlYWs7Y2FzZSAzMzplLnNoaWZ0S2V5P3MudHlwZT0yOnMua2V5PWkuQzAuRVNDKyJbNX4iO2JyZWFrO2Nhc2UgMzQ6ZS5zaGlmdEtleT9zLnR5cGU9MzpzLmtleT1pLkMwLkVTQysiWzZ+IjticmVhaztjYXNlIDExMjpzLmtleT1hP2kuQzAuRVNDKyJbMTsiKyhhKzEpKyJQIjppLkMwLkVTQysiT1AiO2JyZWFrO2Nhc2UgMTEzOnMua2V5PWE/aS5DMC5FU0MrIlsxOyIrKGErMSkrIlEiOmkuQzAuRVNDKyJPUSI7YnJlYWs7Y2FzZSAxMTQ6cy5rZXk9YT9pLkMwLkVTQysiWzE7IisoYSsxKSsiUiI6aS5DMC5FU0MrIk9SIjticmVhaztjYXNlIDExNTpzLmtleT1hP2kuQzAuRVNDKyJbMTsiKyhhKzEpKyJTIjppLkMwLkVTQysiT1MiO2JyZWFrO2Nhc2UgMTE2OnMua2V5PWE/aS5DMC5FU0MrIlsxNTsiKyhhKzEpKyJ+IjppLkMwLkVTQysiWzE1fiI7YnJlYWs7Y2FzZSAxMTc6cy5rZXk9YT9pLkMwLkVTQysiWzE3OyIrKGErMSkrIn4iOmkuQzAuRVNDKyJbMTd+IjticmVhaztjYXNlIDExODpzLmtleT1hP2kuQzAuRVNDKyJbMTg7IisoYSsxKSsifiI6aS5DMC5FU0MrIlsxOH4iO2JyZWFrO2Nhc2UgMTE5OnMua2V5PWE/aS5DMC5FU0MrIlsxOTsiKyhhKzEpKyJ+IjppLkMwLkVTQysiWzE5fiI7YnJlYWs7Y2FzZSAxMjA6cy5rZXk9YT9pLkMwLkVTQysiWzIwOyIrKGErMSkrIn4iOmkuQzAuRVNDKyJbMjB+IjticmVhaztjYXNlIDEyMTpzLmtleT1hP2kuQzAuRVNDKyJbMjE7IisoYSsxKSsifiI6aS5DMC5FU0MrIlsyMX4iO2JyZWFrO2Nhc2UgMTIyOnMua2V5PWE/aS5DMC5FU0MrIlsyMzsiKyhhKzEpKyJ+IjppLkMwLkVTQysiWzIzfiI7YnJlYWs7Y2FzZSAxMjM6cy5rZXk9YT9pLkMwLkVTQysiWzI0OyIrKGErMSkrIn4iOmkuQzAuRVNDKyJbMjR+IjticmVhaztkZWZhdWx0OmlmKCFlLmN0cmxLZXl8fGUuc2hpZnRLZXl8fGUuYWx0S2V5fHxlLm1ldGFLZXkpaWYociYmIW98fCFlLmFsdEtleXx8ZS5tZXRhS2V5KSFyfHxlLmFsdEtleXx8ZS5jdHJsS2V5fHxlLnNoaWZ0S2V5fHwhZS5tZXRhS2V5P2Uua2V5JiYhZS5jdHJsS2V5JiYhZS5hbHRLZXkmJiFlLm1ldGFLZXkmJmUua2V5Q29kZT49NDgmJjE9PT1lLmtleS5sZW5ndGg/cy5rZXk9ZS5rZXk6ZS5rZXkmJmUuY3RybEtleSYmIl8iPT09ZS5rZXkmJihzLmtleT1pLkMwLlVTKTo2NT09PWUua2V5Q29kZSYmKHMudHlwZT0xKTtlbHNle3ZhciBjPW5bZS5rZXlDb2RlXSxsPW51bGw9PWM/dm9pZCAwOmNbZS5zaGlmdEtleT8xOjBdO2lmKGwpcy5rZXk9aS5DMC5FU0MrbDtlbHNlIGlmKGUua2V5Q29kZT49NjUmJmUua2V5Q29kZTw9OTApe3ZhciB1PWUuY3RybEtleT9lLmtleUNvZGUtNjQ6ZS5rZXlDb2RlKzMyO3Mua2V5PWkuQzAuRVNDK1N0cmluZy5mcm9tQ2hhckNvZGUodSl9fWVsc2UgZS5rZXlDb2RlPj02NSYmZS5rZXlDb2RlPD05MD9zLmtleT1TdHJpbmcuZnJvbUNoYXJDb2RlKGUua2V5Q29kZS02NCk6MzI9PT1lLmtleUNvZGU/cy5rZXk9aS5DMC5OVUw6ZS5rZXlDb2RlPj01MSYmZS5rZXlDb2RlPD01NT9zLmtleT1TdHJpbmcuZnJvbUNoYXJDb2RlKGUua2V5Q29kZS01MSsyNyk6NTY9PT1lLmtleUNvZGU/cy5rZXk9aS5DMC5ERUw6MjE5PT09ZS5rZXlDb2RlP3Mua2V5PWkuQzAuRVNDOjIyMD09PWUua2V5Q29kZT9zLmtleT1pLkMwLkZTOjIyMT09PWUua2V5Q29kZSYmKHMua2V5PWkuQzAuR1MpfXJldHVybiBzfX0sNDgyOihlLHQpPT57T2JqZWN0LmRlZmluZVByb3BlcnR5KHQsIl9fZXNNb2R1bGUiLHt2YWx1ZTohMH0pLHQuVXRmOFRvVXRmMzI9dC5TdHJpbmdUb1V0ZjMyPXQudXRmMzJUb1N0cmluZz10LnN0cmluZ0Zyb21Db2RlUG9pbnQ9dm9pZCAwLHQuc3RyaW5nRnJvbUNvZGVQb2ludD1mdW5jdGlvbihlKXtyZXR1cm4gZT42NTUzNT8oZS09NjU1MzYsU3RyaW5nLmZyb21DaGFyQ29kZSg1NTI5NisoZT4+MTApKStTdHJpbmcuZnJvbUNoYXJDb2RlKGUlMTAyNCs1NjMyMCkpOlN0cmluZy5mcm9tQ2hhckNvZGUoZSl9LHQudXRmMzJUb1N0cmluZz1mdW5jdGlvbihlLHQscil7dm9pZCAwPT09dCYmKHQ9MCksdm9pZCAwPT09ciYmKHI9ZS5sZW5ndGgpO2Zvcih2YXIgaT0iIixuPXQ7bjxyOysrbil7dmFyIG89ZVtuXTtvPjY1NTM1PyhvLT02NTUzNixpKz1TdHJpbmcuZnJvbUNoYXJDb2RlKDU1Mjk2KyhvPj4xMCkpK1N0cmluZy5mcm9tQ2hhckNvZGUobyUxMDI0KzU2MzIwKSk6aSs9U3RyaW5nLmZyb21DaGFyQ29kZShvKX1yZXR1cm4gaX07dmFyIHI9ZnVuY3Rpb24oKXtmdW5jdGlvbiBlKCl7dGhpcy5faW50ZXJpbT0wfXJldHVybiBlLnByb3RvdHlwZS5jbGVhcj1mdW5jdGlvbigpe3RoaXMuX2ludGVyaW09MH0sZS5wcm90b3R5cGUuZGVjb2RlPWZ1bmN0aW9uKGUsdCl7dmFyIHI9ZS5sZW5ndGg7aWYoIXIpcmV0dXJuIDA7dmFyIGk9MCxuPTA7dGhpcy5faW50ZXJpbSYmKDU2MzIwPD0oYT1lLmNoYXJDb2RlQXQobisrKSkmJmE8PTU3MzQzP3RbaSsrXT0xMDI0Kih0aGlzLl9pbnRlcmltLTU1Mjk2KSthLTU2MzIwKzY1NTM2Oih0W2krK109dGhpcy5faW50ZXJpbSx0W2krK109YSksdGhpcy5faW50ZXJpbT0wKTtmb3IodmFyIG89bjtvPHI7KytvKXt2YXIgcz1lLmNoYXJDb2RlQXQobyk7aWYoNTUyOTY8PXMmJnM8PTU2MzE5KXtpZigrK28+PXIpcmV0dXJuIHRoaXMuX2ludGVyaW09cyxpO3ZhciBhOzU2MzIwPD0oYT1lLmNoYXJDb2RlQXQobykpJiZhPD01NzM0Mz90W2krK109MTAyNCoocy01NTI5NikrYS01NjMyMCs2NTUzNjoodFtpKytdPXMsdFtpKytdPWEpfWVsc2UgNjUyNzkhPT1zJiYodFtpKytdPXMpfXJldHVybiBpfSxlfSgpO3QuU3RyaW5nVG9VdGYzMj1yO3ZhciBpPWZ1bmN0aW9uKCl7ZnVuY3Rpb24gZSgpe3RoaXMuaW50ZXJpbT1uZXcgVWludDhBcnJheSgzKX1yZXR1cm4gZS5wcm90b3R5cGUuY2xlYXI9ZnVuY3Rpb24oKXt0aGlzLmludGVyaW0uZmlsbCgwKX0sZS5wcm90b3R5cGUuZGVjb2RlPWZ1bmN0aW9uKGUsdCl7dmFyIHI9ZS5sZW5ndGg7aWYoIXIpcmV0dXJuIDA7dmFyIGksbixvLHMsYT0wLGM9MCxsPTA7aWYodGhpcy5pbnRlcmltWzBdKXt2YXIgdT0hMSxoPXRoaXMuaW50ZXJpbVswXTtoJj0xOTI9PSgyMjQmaCk/MzE6MjI0PT0oMjQwJmgpPzE1Ojc7Zm9yKHZhciBmPTAsXz12b2lkIDA7KF89NjMmdGhpcy5pbnRlcmltWysrZl0pJiZmPDQ7KWg8PD02LGh8PV87Zm9yKHZhciBkPTE5Mj09KDIyNCZ0aGlzLmludGVyaW1bMF0pPzI6MjI0PT0oMjQwJnRoaXMuaW50ZXJpbVswXSk/Mzo0LHA9ZC1mO2w8cDspe2lmKGw+PXIpcmV0dXJuIDA7aWYoMTI4IT0oMTkyJihfPWVbbCsrXSkpKXtsLS0sdT0hMDticmVha310aGlzLmludGVyaW1bZisrXT1fLGg8PD02LGh8PTYzJl99dXx8KDI9PT1kP2g8MTI4P2wtLTp0W2ErK109aDozPT09ZD9oPDIwNDh8fGg+PTU1Mjk2JiZoPD01NzM0M3x8NjUyNzk9PT1ofHwodFthKytdPWgpOmg8NjU1MzZ8fGg+MTExNDExMXx8KHRbYSsrXT1oKSksdGhpcy5pbnRlcmltLmZpbGwoMCl9Zm9yKHZhciB2PXItNCxnPWw7ZzxyOyl7Zm9yKDshKCEoZzx2KXx8MTI4JihpPWVbZ10pfHwxMjgmKG49ZVtnKzFdKXx8MTI4JihvPWVbZysyXSl8fDEyOCYocz1lW2crM10pKTspdFthKytdPWksdFthKytdPW4sdFthKytdPW8sdFthKytdPXMsZys9NDtpZigoaT1lW2crK10pPDEyOCl0W2ErK109aTtlbHNlIGlmKDE5Mj09KDIyNCZpKSl7aWYoZz49cilyZXR1cm4gdGhpcy5pbnRlcmltWzBdPWksYTtpZigxMjghPSgxOTImKG49ZVtnKytdKSkpe2ctLTtjb250aW51ZX1pZigoYz0oMzEmaSk8PDZ8NjMmbik8MTI4KXtnLS07Y29udGludWV9dFthKytdPWN9ZWxzZSBpZigyMjQ9PSgyNDAmaSkpe2lmKGc+PXIpcmV0dXJuIHRoaXMuaW50ZXJpbVswXT1pLGE7aWYoMTI4IT0oMTkyJihuPWVbZysrXSkpKXtnLS07Y29udGludWV9aWYoZz49cilyZXR1cm4gdGhpcy5pbnRlcmltWzBdPWksdGhpcy5pbnRlcmltWzFdPW4sYTtpZigxMjghPSgxOTImKG89ZVtnKytdKSkpe2ctLTtjb250aW51ZX1pZigoYz0oMTUmaSk8PDEyfCg2MyZuKTw8Nnw2MyZvKTwyMDQ4fHxjPj01NTI5NiYmYzw9NTczNDN8fDY1Mjc5PT09Yyljb250aW51ZTt0W2ErK109Y31lbHNlIGlmKDI0MD09KDI0OCZpKSl7aWYoZz49cilyZXR1cm4gdGhpcy5pbnRlcmltWzBdPWksYTtpZigxMjghPSgxOTImKG49ZVtnKytdKSkpe2ctLTtjb250aW51ZX1pZihnPj1yKXJldHVybiB0aGlzLmludGVyaW1bMF09aSx0aGlzLmludGVyaW1bMV09bixhO2lmKDEyOCE9KDE5MiYobz1lW2crK10pKSl7Zy0tO2NvbnRpbnVlfWlmKGc+PXIpcmV0dXJuIHRoaXMuaW50ZXJpbVswXT1pLHRoaXMuaW50ZXJpbVsxXT1uLHRoaXMuaW50ZXJpbVsyXT1vLGE7aWYoMTI4IT0oMTkyJihzPWVbZysrXSkpKXtnLS07Y29udGludWV9aWYoKGM9KDcmaSk8PDE4fCg2MyZuKTw8MTJ8KDYzJm8pPDw2fDYzJnMpPDY1NTM2fHxjPjExMTQxMTEpY29udGludWU7dFthKytdPWN9fXJldHVybiBhfSxlfSgpO3QuVXRmOFRvVXRmMzI9aX0sMjI1OihlLHQscik9PntPYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC5Vbmljb2RlVjY9dm9pZCAwO3ZhciBpLG49cig4MjczKSxvPVtbNzY4LDg3OV0sWzExNTUsMTE1OF0sWzExNjAsMTE2MV0sWzE0MjUsMTQ2OV0sWzE0NzEsMTQ3MV0sWzE0NzMsMTQ3NF0sWzE0NzYsMTQ3N10sWzE0NzksMTQ3OV0sWzE1MzYsMTUzOV0sWzE1NTIsMTU1N10sWzE2MTEsMTYzMF0sWzE2NDgsMTY0OF0sWzE3NTAsMTc2NF0sWzE3NjcsMTc2OF0sWzE3NzAsMTc3M10sWzE4MDcsMTgwN10sWzE4MDksMTgwOV0sWzE4NDAsMTg2Nl0sWzE5NTgsMTk2OF0sWzIwMjcsMjAzNV0sWzIzMDUsMjMwNl0sWzIzNjQsMjM2NF0sWzIzNjksMjM3Nl0sWzIzODEsMjM4MV0sWzIzODUsMjM4OF0sWzI0MDIsMjQwM10sWzI0MzMsMjQzM10sWzI0OTIsMjQ5Ml0sWzI0OTcsMjUwMF0sWzI1MDksMjUwOV0sWzI1MzAsMjUzMV0sWzI1NjEsMjU2Ml0sWzI2MjAsMjYyMF0sWzI2MjUsMjYyNl0sWzI2MzEsMjYzMl0sWzI2MzUsMjYzN10sWzI2NzIsMjY3M10sWzI2ODksMjY5MF0sWzI3NDgsMjc0OF0sWzI3NTMsMjc1N10sWzI3NTksMjc2MF0sWzI3NjUsMjc2NV0sWzI3ODYsMjc4N10sWzI4MTcsMjgxN10sWzI4NzYsMjg3Nl0sWzI4NzksMjg3OV0sWzI4ODEsMjg4M10sWzI4OTMsMjg5M10sWzI5MDIsMjkwMl0sWzI5NDYsMjk0Nl0sWzMwMDgsMzAwOF0sWzMwMjEsMzAyMV0sWzMxMzQsMzEzNl0sWzMxNDIsMzE0NF0sWzMxNDYsMzE0OV0sWzMxNTcsMzE1OF0sWzMyNjAsMzI2MF0sWzMyNjMsMzI2M10sWzMyNzAsMzI3MF0sWzMyNzYsMzI3N10sWzMyOTgsMzI5OV0sWzMzOTMsMzM5NV0sWzM0MDUsMzQwNV0sWzM1MzAsMzUzMF0sWzM1MzgsMzU0MF0sWzM1NDIsMzU0Ml0sWzM2MzMsMzYzM10sWzM2MzYsMzY0Ml0sWzM2NTUsMzY2Ml0sWzM3NjEsMzc2MV0sWzM3NjQsMzc2OV0sWzM3NzEsMzc3Ml0sWzM3ODQsMzc4OV0sWzM4NjQsMzg2NV0sWzM4OTMsMzg5M10sWzM4OTUsMzg5NV0sWzM4OTcsMzg5N10sWzM5NTMsMzk2Nl0sWzM5NjgsMzk3Ml0sWzM5NzQsMzk3NV0sWzM5ODQsMzk5MV0sWzM5OTMsNDAyOF0sWzQwMzgsNDAzOF0sWzQxNDEsNDE0NF0sWzQxNDYsNDE0Nl0sWzQxNTAsNDE1MV0sWzQxNTMsNDE1M10sWzQxODQsNDE4NV0sWzQ0NDgsNDYwN10sWzQ5NTksNDk1OV0sWzU5MDYsNTkwOF0sWzU5MzgsNTk0MF0sWzU5NzAsNTk3MV0sWzYwMDIsNjAwM10sWzYwNjgsNjA2OV0sWzYwNzEsNjA3N10sWzYwODYsNjA4Nl0sWzYwODksNjA5OV0sWzYxMDksNjEwOV0sWzYxNTUsNjE1N10sWzYzMTMsNjMxM10sWzY0MzIsNjQzNF0sWzY0MzksNjQ0MF0sWzY0NTAsNjQ1MF0sWzY0NTcsNjQ1OV0sWzY2NzksNjY4MF0sWzY5MTIsNjkxNV0sWzY5NjQsNjk2NF0sWzY5NjYsNjk3MF0sWzY5NzIsNjk3Ml0sWzY5NzgsNjk3OF0sWzcwMTksNzAyN10sWzc2MTYsNzYyNl0sWzc2NzgsNzY3OV0sWzgyMDMsODIwN10sWzgyMzQsODIzOF0sWzgyODgsODI5MV0sWzgyOTgsODMwM10sWzg0MDAsODQzMV0sWzEyMzMwLDEyMzM1XSxbMTI0NDEsMTI0NDJdLFs0MzAxNCw0MzAxNF0sWzQzMDE5LDQzMDE5XSxbNDMwNDUsNDMwNDZdLFs2NDI4Niw2NDI4Nl0sWzY1MDI0LDY1MDM5XSxbNjUwNTYsNjUwNTldLFs2NTI3OSw2NTI3OV0sWzY1NTI5LDY1NTMxXV0scz1bWzY4MDk3LDY4MDk5XSxbNjgxMDEsNjgxMDJdLFs2ODEwOCw2ODExMV0sWzY4MTUyLDY4MTU0XSxbNjgxNTksNjgxNTldLFsxMTkxNDMsMTE5MTQ1XSxbMTE5MTU1LDExOTE3MF0sWzExOTE3MywxMTkxNzldLFsxMTkyMTAsMTE5MjEzXSxbMTE5MzYyLDExOTM2NF0sWzkxNzUwNSw5MTc1MDVdLFs5MTc1MzYsOTE3NjMxXSxbOTE3NzYwLDkxNzk5OV1dLGE9ZnVuY3Rpb24oKXtmdW5jdGlvbiBlKCl7aWYodGhpcy52ZXJzaW9uPSI2IiwhaSl7aT1uZXcgVWludDhBcnJheSg2NTUzNiksKDAsbi5maWxsKShpLDEpLGlbMF09MCwoMCxuLmZpbGwpKGksMCwxLDMyKSwoMCxuLmZpbGwpKGksMCwxMjcsMTYwKSwoMCxuLmZpbGwpKGksMiw0MzUyLDQ0NDgpLGlbOTAwMV09MixpWzkwMDJdPTIsKDAsbi5maWxsKShpLDIsMTE5MDQsNDIxOTIpLGlbMTIzNTFdPTEsKDAsbi5maWxsKShpLDIsNDQwMzIsNTUyMDQpLCgwLG4uZmlsbCkoaSwyLDYzNzQ0LDY0MjU2KSwoMCxuLmZpbGwpKGksMiw2NTA0MCw2NTA1MCksKDAsbi5maWxsKShpLDIsNjUwNzIsNjUxMzYpLCgwLG4uZmlsbCkoaSwyLDY1MjgwLDY1Mzc3KSwoMCxuLmZpbGwpKGksMiw2NTUwNCw2NTUxMSk7Zm9yKHZhciBlPTA7ZTxvLmxlbmd0aDsrK2UpKDAsbi5maWxsKShpLDAsb1tlXVswXSxvW2VdWzFdKzEpfX1yZXR1cm4gZS5wcm90b3R5cGUud2N3aWR0aD1mdW5jdGlvbihlKXtyZXR1cm4gZTwzMj8wOmU8MTI3PzE6ZTw2NTUzNj9pW2VdOmZ1bmN0aW9uKGUsdCl7dmFyIHIsaT0wLG49dC5sZW5ndGgtMTtpZihlPHRbMF1bMF18fGU+dFtuXVsxXSlyZXR1cm4hMTtmb3IoO24+PWk7KWlmKGU+dFtyPWkrbj4+MV1bMV0paT1yKzE7ZWxzZXtpZighKGU8dFtyXVswXSkpcmV0dXJuITA7bj1yLTF9cmV0dXJuITF9KGUscyk/MDplPj0xMzEwNzImJmU8PTE5NjYwNXx8ZT49MTk2NjA4JiZlPD0yNjIxNDE/MjoxfSxlfSgpO3QuVW5pY29kZVY2PWF9LDU5ODE6KGUsdCk9PntPYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC5Xcml0ZUJ1ZmZlcj12b2lkIDA7dmFyIHI9InVuZGVmaW5lZCI9PXR5cGVvZiBxdWV1ZU1pY3JvdGFzaz9mdW5jdGlvbihlKXtQcm9taXNlLnJlc29sdmUoKS50aGVuKGUpfTpxdWV1ZU1pY3JvdGFzayxpPWZ1bmN0aW9uKCl7ZnVuY3Rpb24gZShlKXt0aGlzLl9hY3Rpb249ZSx0aGlzLl93cml0ZUJ1ZmZlcj1bXSx0aGlzLl9jYWxsYmFja3M9W10sdGhpcy5fcGVuZGluZ0RhdGE9MCx0aGlzLl9idWZmZXJPZmZzZXQ9MCx0aGlzLl9pc1N5bmNXcml0aW5nPSExLHRoaXMuX3N5bmNDYWxscz0wfXJldHVybiBlLnByb3RvdHlwZS53cml0ZVN5bmM9ZnVuY3Rpb24oZSx0KXtpZih2b2lkIDAhPT10JiZ0aGlzLl9zeW5jQ2FsbHM+dCl0aGlzLl9zeW5jQ2FsbHM9MDtlbHNlIGlmKHRoaXMuX3BlbmRpbmdEYXRhKz1lLmxlbmd0aCx0aGlzLl93cml0ZUJ1ZmZlci5wdXNoKGUpLHRoaXMuX2NhbGxiYWNrcy5wdXNoKHZvaWQgMCksdGhpcy5fc3luY0NhbGxzKyssIXRoaXMuX2lzU3luY1dyaXRpbmcpe3ZhciByO2Zvcih0aGlzLl9pc1N5bmNXcml0aW5nPSEwO3I9dGhpcy5fd3JpdGVCdWZmZXIuc2hpZnQoKTspe3RoaXMuX2FjdGlvbihyKTt2YXIgaT10aGlzLl9jYWxsYmFja3Muc2hpZnQoKTtpJiZpKCl9dGhpcy5fcGVuZGluZ0RhdGE9MCx0aGlzLl9idWZmZXJPZmZzZXQ9MjE0NzQ4MzY0Nyx0aGlzLl9pc1N5bmNXcml0aW5nPSExLHRoaXMuX3N5bmNDYWxscz0wfX0sZS5wcm90b3R5cGUud3JpdGU9ZnVuY3Rpb24oZSx0KXt2YXIgcj10aGlzO2lmKHRoaXMuX3BlbmRpbmdEYXRhPjVlNyl0aHJvdyBuZXcgRXJyb3IoIndyaXRlIGRhdGEgZGlzY2FyZGVkLCB1c2UgZmxvdyBjb250cm9sIHRvIGF2b2lkIGxvc2luZyBkYXRhIik7dGhpcy5fd3JpdGVCdWZmZXIubGVuZ3RofHwodGhpcy5fYnVmZmVyT2Zmc2V0PTAsc2V0VGltZW91dCgoZnVuY3Rpb24oKXtyZXR1cm4gci5faW5uZXJXcml0ZSgpfSkpKSx0aGlzLl9wZW5kaW5nRGF0YSs9ZS5sZW5ndGgsdGhpcy5fd3JpdGVCdWZmZXIucHVzaChlKSx0aGlzLl9jYWxsYmFja3MucHVzaCh0KX0sZS5wcm90b3R5cGUuX2lubmVyV3JpdGU9ZnVuY3Rpb24oZSx0KXt2YXIgaT10aGlzO3ZvaWQgMD09PWUmJihlPTApLHZvaWQgMD09PXQmJih0PSEwKTtmb3IodmFyIG49ZXx8RGF0ZS5ub3coKTt0aGlzLl93cml0ZUJ1ZmZlci5sZW5ndGg+dGhpcy5fYnVmZmVyT2Zmc2V0Oyl7dmFyIG89dGhpcy5fd3JpdGVCdWZmZXJbdGhpcy5fYnVmZmVyT2Zmc2V0XSxzPXRoaXMuX2FjdGlvbihvLHQpO2lmKHMpcmV0dXJuIHZvaWQgcy5jYXRjaCgoZnVuY3Rpb24oZSl7cmV0dXJuIHIoKGZ1bmN0aW9uKCl7dGhyb3cgZX0pKSxQcm9taXNlLnJlc29sdmUoITEpfSkpLnRoZW4oKGZ1bmN0aW9uKGUpe3JldHVybiBEYXRlLm5vdygpLW4+PTEyP3NldFRpbWVvdXQoKGZ1bmN0aW9uKCl7cmV0dXJuIGkuX2lubmVyV3JpdGUoMCxlKX0pKTppLl9pbm5lcldyaXRlKG4sZSl9KSk7dmFyIGE9dGhpcy5fY2FsbGJhY2tzW3RoaXMuX2J1ZmZlck9mZnNldF07aWYoYSYmYSgpLHRoaXMuX2J1ZmZlck9mZnNldCsrLHRoaXMuX3BlbmRpbmdEYXRhLT1vLmxlbmd0aCxEYXRlLm5vdygpLW4+PTEyKWJyZWFrfXRoaXMuX3dyaXRlQnVmZmVyLmxlbmd0aD50aGlzLl9idWZmZXJPZmZzZXQ/KHRoaXMuX2J1ZmZlck9mZnNldD41MCYmKHRoaXMuX3dyaXRlQnVmZmVyPXRoaXMuX3dyaXRlQnVmZmVyLnNsaWNlKHRoaXMuX2J1ZmZlck9mZnNldCksdGhpcy5fY2FsbGJhY2tzPXRoaXMuX2NhbGxiYWNrcy5zbGljZSh0aGlzLl9idWZmZXJPZmZzZXQpLHRoaXMuX2J1ZmZlck9mZnNldD0wKSxzZXRUaW1lb3V0KChmdW5jdGlvbigpe3JldHVybiBpLl9pbm5lcldyaXRlKCl9KSkpOih0aGlzLl93cml0ZUJ1ZmZlci5sZW5ndGg9MCx0aGlzLl9jYWxsYmFja3MubGVuZ3RoPTAsdGhpcy5fcGVuZGluZ0RhdGE9MCx0aGlzLl9idWZmZXJPZmZzZXQ9MCl9LGV9KCk7dC5Xcml0ZUJ1ZmZlcj1pfSw1OTQxOihlLHQpPT57T2JqZWN0LmRlZmluZVByb3BlcnR5KHQsIl9fZXNNb2R1bGUiLHt2YWx1ZTohMH0pLHQudG9SZ2JTdHJpbmc9dC5wYXJzZUNvbG9yPXZvaWQgMDt2YXIgcj0vXihbXGRhLWZdezF9KVwvKFtcZGEtZl17MX0pXC8oW1xkYS1mXXsxfSkkfF4oW1xkYS1mXXsyfSlcLyhbXGRhLWZdezJ9KVwvKFtcZGEtZl17Mn0pJHxeKFtcZGEtZl17M30pXC8oW1xkYS1mXXszfSlcLyhbXGRhLWZdezN9KSR8XihbXGRhLWZdezR9KVwvKFtcZGEtZl17NH0pXC8oW1xkYS1mXXs0fSkkLyxpPS9eW1xkYS1mXSskLztmdW5jdGlvbiBuKGUsdCl7dmFyIHI9ZS50b1N0cmluZygxNiksaT1yLmxlbmd0aDwyPyIwIityOnI7c3dpdGNoKHQpe2Nhc2UgNDpyZXR1cm4gclswXTtjYXNlIDg6cmV0dXJuIGk7Y2FzZSAxMjpyZXR1cm4oaStpKS5zbGljZSgwLDMpO2RlZmF1bHQ6cmV0dXJuIGkraX19dC5wYXJzZUNvbG9yPWZ1bmN0aW9uKGUpe2lmKGUpe3ZhciB0PWUudG9Mb3dlckNhc2UoKTtpZigwPT09dC5pbmRleE9mKCJyZ2I6Iikpe3Q9dC5zbGljZSg0KTt2YXIgbj1yLmV4ZWModCk7aWYobil7dmFyIG89blsxXT8xNTpuWzRdPzI1NTpuWzddPzQwOTU6NjU1MzU7cmV0dXJuW01hdGgucm91bmQocGFyc2VJbnQoblsxXXx8bls0XXx8bls3XXx8blsxMF0sMTYpL28qMjU1KSxNYXRoLnJvdW5kKHBhcnNlSW50KG5bMl18fG5bNV18fG5bOF18fG5bMTFdLDE2KS9vKjI1NSksTWF0aC5yb3VuZChwYXJzZUludChuWzNdfHxuWzZdfHxuWzldfHxuWzEyXSwxNikvbyoyNTUpXX19ZWxzZSBpZigwPT09dC5pbmRleE9mKCIjIikmJih0PXQuc2xpY2UoMSksaS5leGVjKHQpJiZbMyw2LDksMTJdLmluY2x1ZGVzKHQubGVuZ3RoKSkpe2Zvcih2YXIgcz10Lmxlbmd0aC8zLGE9WzAsMCwwXSxjPTA7YzwzOysrYyl7dmFyIGw9cGFyc2VJbnQodC5zbGljZShzKmMscypjK3MpLDE2KTthW2NdPTE9PT1zP2w8PDQ6Mj09PXM/bDozPT09cz9sPj40Omw+Pjh9cmV0dXJuIGF9fX0sdC50b1JnYlN0cmluZz1mdW5jdGlvbihlLHQpe3ZvaWQgMD09PXQmJih0PTE2KTt2YXIgcj1lWzBdLGk9ZVsxXSxvPWVbMl07cmV0dXJuInJnYjoiK24ocix0KSsiLyIrbihpLHQpKyIvIituKG8sdCl9fSw1NzcwOihlLHQpPT57T2JqZWN0LmRlZmluZVByb3BlcnR5KHQsIl9fZXNNb2R1bGUiLHt2YWx1ZTohMH0pLHQuUEFZTE9BRF9MSU1JVD12b2lkIDAsdC5QQVlMT0FEX0xJTUlUPTFlN30sNjM1MTooZSx0LHIpPT57T2JqZWN0LmRlZmluZVByb3BlcnR5KHQsIl9fZXNNb2R1bGUiLHt2YWx1ZTohMH0pLHQuRGNzSGFuZGxlcj10LkRjc1BhcnNlcj12b2lkIDA7dmFyIGk9cig0ODIpLG49cig4NzQyKSxvPXIoNTc3MCkscz1bXSxhPWZ1bmN0aW9uKCl7ZnVuY3Rpb24gZSgpe3RoaXMuX2hhbmRsZXJzPU9iamVjdC5jcmVhdGUobnVsbCksdGhpcy5fYWN0aXZlPXMsdGhpcy5faWRlbnQ9MCx0aGlzLl9oYW5kbGVyRmI9ZnVuY3Rpb24oKXt9LHRoaXMuX3N0YWNrPXtwYXVzZWQ6ITEsbG9vcFBvc2l0aW9uOjAsZmFsbFRocm91Z2g6ITF9fXJldHVybiBlLnByb3RvdHlwZS5kaXNwb3NlPWZ1bmN0aW9uKCl7dGhpcy5faGFuZGxlcnM9T2JqZWN0LmNyZWF0ZShudWxsKSx0aGlzLl9oYW5kbGVyRmI9ZnVuY3Rpb24oKXt9LHRoaXMuX2FjdGl2ZT1zfSxlLnByb3RvdHlwZS5yZWdpc3RlckhhbmRsZXI9ZnVuY3Rpb24oZSx0KXt2b2lkIDA9PT10aGlzLl9oYW5kbGVyc1tlXSYmKHRoaXMuX2hhbmRsZXJzW2VdPVtdKTt2YXIgcj10aGlzLl9oYW5kbGVyc1tlXTtyZXR1cm4gci5wdXNoKHQpLHtkaXNwb3NlOmZ1bmN0aW9uKCl7dmFyIGU9ci5pbmRleE9mKHQpOy0xIT09ZSYmci5zcGxpY2UoZSwxKX19fSxlLnByb3RvdHlwZS5jbGVhckhhbmRsZXI9ZnVuY3Rpb24oZSl7dGhpcy5faGFuZGxlcnNbZV0mJmRlbGV0ZSB0aGlzLl9oYW5kbGVyc1tlXX0sZS5wcm90b3R5cGUuc2V0SGFuZGxlckZhbGxiYWNrPWZ1bmN0aW9uKGUpe3RoaXMuX2hhbmRsZXJGYj1lfSxlLnByb3RvdHlwZS5yZXNldD1mdW5jdGlvbigpe2lmKHRoaXMuX2FjdGl2ZS5sZW5ndGgpZm9yKHZhciBlPXRoaXMuX3N0YWNrLnBhdXNlZD90aGlzLl9zdGFjay5sb29wUG9zaXRpb24tMTp0aGlzLl9hY3RpdmUubGVuZ3RoLTE7ZT49MDstLWUpdGhpcy5fYWN0aXZlW2VdLnVuaG9vayghMSk7dGhpcy5fc3RhY2sucGF1c2VkPSExLHRoaXMuX2FjdGl2ZT1zLHRoaXMuX2lkZW50PTB9LGUucHJvdG90eXBlLmhvb2s9ZnVuY3Rpb24oZSx0KXtpZih0aGlzLnJlc2V0KCksdGhpcy5faWRlbnQ9ZSx0aGlzLl9hY3RpdmU9dGhpcy5faGFuZGxlcnNbZV18fHMsdGhpcy5fYWN0aXZlLmxlbmd0aClmb3IodmFyIHI9dGhpcy5fYWN0aXZlLmxlbmd0aC0xO3I+PTA7ci0tKXRoaXMuX2FjdGl2ZVtyXS5ob29rKHQpO2Vsc2UgdGhpcy5faGFuZGxlckZiKHRoaXMuX2lkZW50LCJIT09LIix0KX0sZS5wcm90b3R5cGUucHV0PWZ1bmN0aW9uKGUsdCxyKXtpZih0aGlzLl9hY3RpdmUubGVuZ3RoKWZvcih2YXIgbj10aGlzLl9hY3RpdmUubGVuZ3RoLTE7bj49MDtuLS0pdGhpcy5fYWN0aXZlW25dLnB1dChlLHQscik7ZWxzZSB0aGlzLl9oYW5kbGVyRmIodGhpcy5faWRlbnQsIlBVVCIsKDAsaS51dGYzMlRvU3RyaW5nKShlLHQscikpfSxlLnByb3RvdHlwZS51bmhvb2s9ZnVuY3Rpb24oZSx0KXtpZih2b2lkIDA9PT10JiYodD0hMCksdGhpcy5fYWN0aXZlLmxlbmd0aCl7dmFyIHI9ITEsaT10aGlzLl9hY3RpdmUubGVuZ3RoLTEsbj0hMTtpZih0aGlzLl9zdGFjay5wYXVzZWQmJihpPXRoaXMuX3N0YWNrLmxvb3BQb3NpdGlvbi0xLHI9dCxuPXRoaXMuX3N0YWNrLmZhbGxUaHJvdWdoLHRoaXMuX3N0YWNrLnBhdXNlZD0hMSksIW4mJiExPT09cil7Zm9yKDtpPj0wJiYhMCE9PShyPXRoaXMuX2FjdGl2ZVtpXS51bmhvb2soZSkpO2ktLSlpZihyIGluc3RhbmNlb2YgUHJvbWlzZSlyZXR1cm4gdGhpcy5fc3RhY2sucGF1c2VkPSEwLHRoaXMuX3N0YWNrLmxvb3BQb3NpdGlvbj1pLHRoaXMuX3N0YWNrLmZhbGxUaHJvdWdoPSExLHI7aS0tfWZvcig7aT49MDtpLS0paWYoKHI9dGhpcy5fYWN0aXZlW2ldLnVuaG9vayghMSkpaW5zdGFuY2VvZiBQcm9taXNlKXJldHVybiB0aGlzLl9zdGFjay5wYXVzZWQ9ITAsdGhpcy5fc3RhY2subG9vcFBvc2l0aW9uPWksdGhpcy5fc3RhY2suZmFsbFRocm91Z2g9ITAscn1lbHNlIHRoaXMuX2hhbmRsZXJGYih0aGlzLl9pZGVudCwiVU5IT09LIixlKTt0aGlzLl9hY3RpdmU9cyx0aGlzLl9pZGVudD0wfSxlfSgpO3QuRGNzUGFyc2VyPWE7dmFyIGM9bmV3IG4uUGFyYW1zO2MuYWRkUGFyYW0oMCk7dmFyIGw9ZnVuY3Rpb24oKXtmdW5jdGlvbiBlKGUpe3RoaXMuX2hhbmRsZXI9ZSx0aGlzLl9kYXRhPSIiLHRoaXMuX3BhcmFtcz1jLHRoaXMuX2hpdExpbWl0PSExfXJldHVybiBlLnByb3RvdHlwZS5ob29rPWZ1bmN0aW9uKGUpe3RoaXMuX3BhcmFtcz1lLmxlbmd0aD4xfHxlLnBhcmFtc1swXT9lLmNsb25lKCk6Yyx0aGlzLl9kYXRhPSIiLHRoaXMuX2hpdExpbWl0PSExfSxlLnByb3RvdHlwZS5wdXQ9ZnVuY3Rpb24oZSx0LHIpe3RoaXMuX2hpdExpbWl0fHwodGhpcy5fZGF0YSs9KDAsaS51dGYzMlRvU3RyaW5nKShlLHQsciksdGhpcy5fZGF0YS5sZW5ndGg+by5QQVlMT0FEX0xJTUlUJiYodGhpcy5fZGF0YT0iIix0aGlzLl9oaXRMaW1pdD0hMCkpfSxlLnByb3RvdHlwZS51bmhvb2s9ZnVuY3Rpb24oZSl7dmFyIHQ9dGhpcyxyPSExO2lmKHRoaXMuX2hpdExpbWl0KXI9ITE7ZWxzZSBpZihlJiYocj10aGlzLl9oYW5kbGVyKHRoaXMuX2RhdGEsdGhpcy5fcGFyYW1zKSlpbnN0YW5jZW9mIFByb21pc2UpcmV0dXJuIHIudGhlbigoZnVuY3Rpb24oZSl7cmV0dXJuIHQuX3BhcmFtcz1jLHQuX2RhdGE9IiIsdC5faGl0TGltaXQ9ITEsZX0pKTtyZXR1cm4gdGhpcy5fcGFyYW1zPWMsdGhpcy5fZGF0YT0iIix0aGlzLl9oaXRMaW1pdD0hMSxyfSxlfSgpO3QuRGNzSGFuZGxlcj1sfSwyMDE1OmZ1bmN0aW9uKGUsdCxyKXt2YXIgaSxuPXRoaXMmJnRoaXMuX19leHRlbmRzfHwoaT1mdW5jdGlvbihlLHQpe3JldHVybiBpPU9iamVjdC5zZXRQcm90b3R5cGVPZnx8e19fcHJvdG9fXzpbXX1pbnN0YW5jZW9mIEFycmF5JiZmdW5jdGlvbihlLHQpe2UuX19wcm90b19fPXR9fHxmdW5jdGlvbihlLHQpe2Zvcih2YXIgciBpbiB0KU9iamVjdC5wcm90b3R5cGUuaGFzT3duUHJvcGVydHkuY2FsbCh0LHIpJiYoZVtyXT10W3JdKX0saShlLHQpfSxmdW5jdGlvbihlLHQpe2lmKCJmdW5jdGlvbiIhPXR5cGVvZiB0JiZudWxsIT09dCl0aHJvdyBuZXcgVHlwZUVycm9yKCJDbGFzcyBleHRlbmRzIHZhbHVlICIrU3RyaW5nKHQpKyIgaXMgbm90IGEgY29uc3RydWN0b3Igb3IgbnVsbCIpO2Z1bmN0aW9uIHIoKXt0aGlzLmNvbnN0cnVjdG9yPWV9aShlLHQpLGUucHJvdG90eXBlPW51bGw9PT10P09iamVjdC5jcmVhdGUodCk6KHIucHJvdG90eXBlPXQucHJvdG90eXBlLG5ldyByKX0pO09iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LCJfX2VzTW9kdWxlIix7dmFsdWU6ITB9KSx0LkVzY2FwZVNlcXVlbmNlUGFyc2VyPXQuVlQ1MDBfVFJBTlNJVElPTl9UQUJMRT10LlRyYW5zaXRpb25UYWJsZT12b2lkIDA7dmFyIG89cig4NDQpLHM9cig4MjczKSxhPXIoODc0MiksYz1yKDYyNDIpLGw9cig2MzUxKSx1PWZ1bmN0aW9uKCl7ZnVuY3Rpb24gZShlKXt0aGlzLnRhYmxlPW5ldyBVaW50OEFycmF5KGUpfXJldHVybiBlLnByb3RvdHlwZS5zZXREZWZhdWx0PWZ1bmN0aW9uKGUsdCl7KDAscy5maWxsKSh0aGlzLnRhYmxlLGU8PDR8dCl9LGUucHJvdG90eXBlLmFkZD1mdW5jdGlvbihlLHQscixpKXt0aGlzLnRhYmxlW3Q8PDh8ZV09cjw8NHxpfSxlLnByb3RvdHlwZS5hZGRNYW55PWZ1bmN0aW9uKGUsdCxyLGkpe2Zvcih2YXIgbj0wO248ZS5sZW5ndGg7bisrKXRoaXMudGFibGVbdDw8OHxlW25dXT1yPDw0fGl9LGV9KCk7dC5UcmFuc2l0aW9uVGFibGU9dTt2YXIgaD0xNjA7dC5WVDUwMF9UUkFOU0lUSU9OX1RBQkxFPWZ1bmN0aW9uKCl7dmFyIGU9bmV3IHUoNDA5NSksdD1BcnJheS5hcHBseShudWxsLEFycmF5KDI1NikpLm1hcCgoZnVuY3Rpb24oZSx0KXtyZXR1cm4gdH0pKSxyPWZ1bmN0aW9uKGUscil7cmV0dXJuIHQuc2xpY2UoZSxyKX0saT1yKDMyLDEyNyksbj1yKDAsMjQpO24ucHVzaCgyNSksbi5wdXNoLmFwcGx5KG4scigyOCwzMikpO3ZhciBvLHM9cigwLDE0KTtmb3IobyBpbiBlLnNldERlZmF1bHQoMSwwKSxlLmFkZE1hbnkoaSwwLDIsMCkscyllLmFkZE1hbnkoWzI0LDI2LDE1MywxNTRdLG8sMywwKSxlLmFkZE1hbnkocigxMjgsMTQ0KSxvLDMsMCksZS5hZGRNYW55KHIoMTQ0LDE1MiksbywzLDApLGUuYWRkKDE1NixvLDAsMCksZS5hZGQoMjcsbywxMSwxKSxlLmFkZCgxNTcsbyw0LDgpLGUuYWRkTWFueShbMTUyLDE1OCwxNTldLG8sMCw3KSxlLmFkZCgxNTUsbywxMSwzKSxlLmFkZCgxNDQsbywxMSw5KTtyZXR1cm4gZS5hZGRNYW55KG4sMCwzLDApLGUuYWRkTWFueShuLDEsMywxKSxlLmFkZCgxMjcsMSwwLDEpLGUuYWRkTWFueShuLDgsMCw4KSxlLmFkZE1hbnkobiwzLDMsMyksZS5hZGQoMTI3LDMsMCwzKSxlLmFkZE1hbnkobiw0LDMsNCksZS5hZGQoMTI3LDQsMCw0KSxlLmFkZE1hbnkobiw2LDMsNiksZS5hZGRNYW55KG4sNSwzLDUpLGUuYWRkKDEyNyw1LDAsNSksZS5hZGRNYW55KG4sMiwzLDIpLGUuYWRkKDEyNywyLDAsMiksZS5hZGQoOTMsMSw0LDgpLGUuYWRkTWFueShpLDgsNSw4KSxlLmFkZCgxMjcsOCw1LDgpLGUuYWRkTWFueShbMTU2LDI3LDI0LDI2LDddLDgsNiwwKSxlLmFkZE1hbnkocigyOCwzMiksOCwwLDgpLGUuYWRkTWFueShbODgsOTQsOTVdLDEsMCw3KSxlLmFkZE1hbnkoaSw3LDAsNyksZS5hZGRNYW55KG4sNywwLDcpLGUuYWRkKDE1Niw3LDAsMCksZS5hZGQoMTI3LDcsMCw3KSxlLmFkZCg5MSwxLDExLDMpLGUuYWRkTWFueShyKDY0LDEyNyksMyw3LDApLGUuYWRkTWFueShyKDQ4LDYwKSwzLDgsNCksZS5hZGRNYW55KFs2MCw2MSw2Miw2M10sMyw5LDQpLGUuYWRkTWFueShyKDQ4LDYwKSw0LDgsNCksZS5hZGRNYW55KHIoNjQsMTI3KSw0LDcsMCksZS5hZGRNYW55KFs2MCw2MSw2Miw2M10sNCwwLDYpLGUuYWRkTWFueShyKDMyLDY0KSw2LDAsNiksZS5hZGQoMTI3LDYsMCw2KSxlLmFkZE1hbnkocig2NCwxMjcpLDYsMCwwKSxlLmFkZE1hbnkocigzMiw0OCksMyw5LDUpLGUuYWRkTWFueShyKDMyLDQ4KSw1LDksNSksZS5hZGRNYW55KHIoNDgsNjQpLDUsMCw2KSxlLmFkZE1hbnkocig2NCwxMjcpLDUsNywwKSxlLmFkZE1hbnkocigzMiw0OCksNCw5LDUpLGUuYWRkTWFueShyKDMyLDQ4KSwxLDksMiksZS5hZGRNYW55KHIoMzIsNDgpLDIsOSwyKSxlLmFkZE1hbnkocig0OCwxMjcpLDIsMTAsMCksZS5hZGRNYW55KHIoNDgsODApLDEsMTAsMCksZS5hZGRNYW55KHIoODEsODgpLDEsMTAsMCksZS5hZGRNYW55KFs4OSw5MCw5Ml0sMSwxMCwwKSxlLmFkZE1hbnkocig5NiwxMjcpLDEsMTAsMCksZS5hZGQoODAsMSwxMSw5KSxlLmFkZE1hbnkobiw5LDAsOSksZS5hZGQoMTI3LDksMCw5KSxlLmFkZE1hbnkocigyOCwzMiksOSwwLDkpLGUuYWRkTWFueShyKDMyLDQ4KSw5LDksMTIpLGUuYWRkTWFueShyKDQ4LDYwKSw5LDgsMTApLGUuYWRkTWFueShbNjAsNjEsNjIsNjNdLDksOSwxMCksZS5hZGRNYW55KG4sMTEsMCwxMSksZS5hZGRNYW55KHIoMzIsMTI4KSwxMSwwLDExKSxlLmFkZE1hbnkocigyOCwzMiksMTEsMCwxMSksZS5hZGRNYW55KG4sMTAsMCwxMCksZS5hZGQoMTI3LDEwLDAsMTApLGUuYWRkTWFueShyKDI4LDMyKSwxMCwwLDEwKSxlLmFkZE1hbnkocig0OCw2MCksMTAsOCwxMCksZS5hZGRNYW55KFs2MCw2MSw2Miw2M10sMTAsMCwxMSksZS5hZGRNYW55KHIoMzIsNDgpLDEwLDksMTIpLGUuYWRkTWFueShuLDEyLDAsMTIpLGUuYWRkKDEyNywxMiwwLDEyKSxlLmFkZE1hbnkocigyOCwzMiksMTIsMCwxMiksZS5hZGRNYW55KHIoMzIsNDgpLDEyLDksMTIpLGUuYWRkTWFueShyKDQ4LDY0KSwxMiwwLDExKSxlLmFkZE1hbnkocig2NCwxMjcpLDEyLDEyLDEzKSxlLmFkZE1hbnkocig2NCwxMjcpLDEwLDEyLDEzKSxlLmFkZE1hbnkocig2NCwxMjcpLDksMTIsMTMpLGUuYWRkTWFueShuLDEzLDEzLDEzKSxlLmFkZE1hbnkoaSwxMywxMywxMyksZS5hZGQoMTI3LDEzLDAsMTMpLGUuYWRkTWFueShbMjcsMTU2LDI0LDI2XSwxMywxNCwwKSxlLmFkZChoLDAsMiwwKSxlLmFkZChoLDgsNSw4KSxlLmFkZChoLDYsMCw2KSxlLmFkZChoLDExLDAsMTEpLGUuYWRkKGgsMTMsMTMsMTMpLGV9KCk7dmFyIGY9ZnVuY3Rpb24oZSl7ZnVuY3Rpb24gcihyKXt2b2lkIDA9PT1yJiYocj10LlZUNTAwX1RSQU5TSVRJT05fVEFCTEUpO3ZhciBpPWUuY2FsbCh0aGlzKXx8dGhpcztyZXR1cm4gaS5fdHJhbnNpdGlvbnM9cixpLl9wYXJzZVN0YWNrPXtzdGF0ZTowLGhhbmRsZXJzOltdLGhhbmRsZXJQb3M6MCx0cmFuc2l0aW9uOjAsY2h1bmtQb3M6MH0saS5pbml0aWFsU3RhdGU9MCxpLmN1cnJlbnRTdGF0ZT1pLmluaXRpYWxTdGF0ZSxpLl9wYXJhbXM9bmV3IGEuUGFyYW1zLGkuX3BhcmFtcy5hZGRQYXJhbSgwKSxpLl9jb2xsZWN0PTAsaS5wcmVjZWRpbmdDb2RlcG9pbnQ9MCxpLl9wcmludEhhbmRsZXJGYj1mdW5jdGlvbihlLHQscil7fSxpLl9leGVjdXRlSGFuZGxlckZiPWZ1bmN0aW9uKGUpe30saS5fY3NpSGFuZGxlckZiPWZ1bmN0aW9uKGUsdCl7fSxpLl9lc2NIYW5kbGVyRmI9ZnVuY3Rpb24oZSl7fSxpLl9lcnJvckhhbmRsZXJGYj1mdW5jdGlvbihlKXtyZXR1cm4gZX0saS5fcHJpbnRIYW5kbGVyPWkuX3ByaW50SGFuZGxlckZiLGkuX2V4ZWN1dGVIYW5kbGVycz1PYmplY3QuY3JlYXRlKG51bGwpLGkuX2NzaUhhbmRsZXJzPU9iamVjdC5jcmVhdGUobnVsbCksaS5fZXNjSGFuZGxlcnM9T2JqZWN0LmNyZWF0ZShudWxsKSxpLl9vc2NQYXJzZXI9bmV3IGMuT3NjUGFyc2VyLGkuX2Rjc1BhcnNlcj1uZXcgbC5EY3NQYXJzZXIsaS5fZXJyb3JIYW5kbGVyPWkuX2Vycm9ySGFuZGxlckZiLGkucmVnaXN0ZXJFc2NIYW5kbGVyKHtmaW5hbDoiXFwifSwoZnVuY3Rpb24oKXtyZXR1cm4hMH0pKSxpfXJldHVybiBuKHIsZSksci5wcm90b3R5cGUuX2lkZW50aWZpZXI9ZnVuY3Rpb24oZSx0KXt2b2lkIDA9PT10JiYodD1bNjQsMTI2XSk7dmFyIHI9MDtpZihlLnByZWZpeCl7aWYoZS5wcmVmaXgubGVuZ3RoPjEpdGhyb3cgbmV3IEVycm9yKCJvbmx5IG9uZSBieXRlIGFzIHByZWZpeCBzdXBwb3J0ZWQiKTtpZigocj1lLnByZWZpeC5jaGFyQ29kZUF0KDApKSYmNjA+cnx8cj42Myl0aHJvdyBuZXcgRXJyb3IoInByZWZpeCBtdXN0IGJlIGluIHJhbmdlIDB4M2MgLi4gMHgzZiIpfWlmKGUuaW50ZXJtZWRpYXRlcyl7aWYoZS5pbnRlcm1lZGlhdGVzLmxlbmd0aD4yKXRocm93IG5ldyBFcnJvcigib25seSB0d28gYnl0ZXMgYXMgaW50ZXJtZWRpYXRlcyBhcmUgc3VwcG9ydGVkIik7Zm9yKHZhciBpPTA7aTxlLmludGVybWVkaWF0ZXMubGVuZ3RoOysraSl7dmFyIG49ZS5pbnRlcm1lZGlhdGVzLmNoYXJDb2RlQXQoaSk7aWYoMzI+bnx8bj40Nyl0aHJvdyBuZXcgRXJyb3IoImludGVybWVkaWF0ZSBtdXN0IGJlIGluIHJhbmdlIDB4MjAgLi4gMHgyZiIpO3I8PD04LHJ8PW59fWlmKDEhPT1lLmZpbmFsLmxlbmd0aCl0aHJvdyBuZXcgRXJyb3IoImZpbmFsIG11c3QgYmUgYSBzaW5nbGUgYnl0ZSIpO3ZhciBvPWUuZmluYWwuY2hhckNvZGVBdCgwKTtpZih0WzBdPm98fG8+dFsxXSl0aHJvdyBuZXcgRXJyb3IoImZpbmFsIG11c3QgYmUgaW4gcmFuZ2UgIit0WzBdKyIgLi4gIit0WzFdKTtyZXR1cm4ocjw8PTgpfG99LHIucHJvdG90eXBlLmlkZW50VG9TdHJpbmc9ZnVuY3Rpb24oZSl7Zm9yKHZhciB0PVtdO2U7KXQucHVzaChTdHJpbmcuZnJvbUNoYXJDb2RlKDI1NSZlKSksZT4+PTg7cmV0dXJuIHQucmV2ZXJzZSgpLmpvaW4oIiIpfSxyLnByb3RvdHlwZS5kaXNwb3NlPWZ1bmN0aW9uKCl7dGhpcy5fY3NpSGFuZGxlcnM9T2JqZWN0LmNyZWF0ZShudWxsKSx0aGlzLl9leGVjdXRlSGFuZGxlcnM9T2JqZWN0LmNyZWF0ZShudWxsKSx0aGlzLl9lc2NIYW5kbGVycz1PYmplY3QuY3JlYXRlKG51bGwpLHRoaXMuX29zY1BhcnNlci5kaXNwb3NlKCksdGhpcy5fZGNzUGFyc2VyLmRpc3Bvc2UoKX0sci5wcm90b3R5cGUuc2V0UHJpbnRIYW5kbGVyPWZ1bmN0aW9uKGUpe3RoaXMuX3ByaW50SGFuZGxlcj1lfSxyLnByb3RvdHlwZS5jbGVhclByaW50SGFuZGxlcj1mdW5jdGlvbigpe3RoaXMuX3ByaW50SGFuZGxlcj10aGlzLl9wcmludEhhbmRsZXJGYn0sci5wcm90b3R5cGUucmVnaXN0ZXJFc2NIYW5kbGVyPWZ1bmN0aW9uKGUsdCl7dmFyIHI9dGhpcy5faWRlbnRpZmllcihlLFs0OCwxMjZdKTt2b2lkIDA9PT10aGlzLl9lc2NIYW5kbGVyc1tyXSYmKHRoaXMuX2VzY0hhbmRsZXJzW3JdPVtdKTt2YXIgaT10aGlzLl9lc2NIYW5kbGVyc1tyXTtyZXR1cm4gaS5wdXNoKHQpLHtkaXNwb3NlOmZ1bmN0aW9uKCl7dmFyIGU9aS5pbmRleE9mKHQpOy0xIT09ZSYmaS5zcGxpY2UoZSwxKX19fSxyLnByb3RvdHlwZS5jbGVhckVzY0hhbmRsZXI9ZnVuY3Rpb24oZSl7dGhpcy5fZXNjSGFuZGxlcnNbdGhpcy5faWRlbnRpZmllcihlLFs0OCwxMjZdKV0mJmRlbGV0ZSB0aGlzLl9lc2NIYW5kbGVyc1t0aGlzLl9pZGVudGlmaWVyKGUsWzQ4LDEyNl0pXX0sci5wcm90b3R5cGUuc2V0RXNjSGFuZGxlckZhbGxiYWNrPWZ1bmN0aW9uKGUpe3RoaXMuX2VzY0hhbmRsZXJGYj1lfSxyLnByb3RvdHlwZS5zZXRFeGVjdXRlSGFuZGxlcj1mdW5jdGlvbihlLHQpe3RoaXMuX2V4ZWN1dGVIYW5kbGVyc1tlLmNoYXJDb2RlQXQoMCldPXR9LHIucHJvdG90eXBlLmNsZWFyRXhlY3V0ZUhhbmRsZXI9ZnVuY3Rpb24oZSl7dGhpcy5fZXhlY3V0ZUhhbmRsZXJzW2UuY2hhckNvZGVBdCgwKV0mJmRlbGV0ZSB0aGlzLl9leGVjdXRlSGFuZGxlcnNbZS5jaGFyQ29kZUF0KDApXX0sci5wcm90b3R5cGUuc2V0RXhlY3V0ZUhhbmRsZXJGYWxsYmFjaz1mdW5jdGlvbihlKXt0aGlzLl9leGVjdXRlSGFuZGxlckZiPWV9LHIucHJvdG90eXBlLnJlZ2lzdGVyQ3NpSGFuZGxlcj1mdW5jdGlvbihlLHQpe3ZhciByPXRoaXMuX2lkZW50aWZpZXIoZSk7dm9pZCAwPT09dGhpcy5fY3NpSGFuZGxlcnNbcl0mJih0aGlzLl9jc2lIYW5kbGVyc1tyXT1bXSk7dmFyIGk9dGhpcy5fY3NpSGFuZGxlcnNbcl07cmV0dXJuIGkucHVzaCh0KSx7ZGlzcG9zZTpmdW5jdGlvbigpe3ZhciBlPWkuaW5kZXhPZih0KTstMSE9PWUmJmkuc3BsaWNlKGUsMSl9fX0sci5wcm90b3R5cGUuY2xlYXJDc2lIYW5kbGVyPWZ1bmN0aW9uKGUpe3RoaXMuX2NzaUhhbmRsZXJzW3RoaXMuX2lkZW50aWZpZXIoZSldJiZkZWxldGUgdGhpcy5fY3NpSGFuZGxlcnNbdGhpcy5faWRlbnRpZmllcihlKV19LHIucHJvdG90eXBlLnNldENzaUhhbmRsZXJGYWxsYmFjaz1mdW5jdGlvbihlKXt0aGlzLl9jc2lIYW5kbGVyRmI9ZX0sci5wcm90b3R5cGUucmVnaXN0ZXJEY3NIYW5kbGVyPWZ1bmN0aW9uKGUsdCl7cmV0dXJuIHRoaXMuX2Rjc1BhcnNlci5yZWdpc3RlckhhbmRsZXIodGhpcy5faWRlbnRpZmllcihlKSx0KX0sci5wcm90b3R5cGUuY2xlYXJEY3NIYW5kbGVyPWZ1bmN0aW9uKGUpe3RoaXMuX2Rjc1BhcnNlci5jbGVhckhhbmRsZXIodGhpcy5faWRlbnRpZmllcihlKSl9LHIucHJvdG90eXBlLnNldERjc0hhbmRsZXJGYWxsYmFjaz1mdW5jdGlvbihlKXt0aGlzLl9kY3NQYXJzZXIuc2V0SGFuZGxlckZhbGxiYWNrKGUpfSxyLnByb3RvdHlwZS5yZWdpc3Rlck9zY0hhbmRsZXI9ZnVuY3Rpb24oZSx0KXtyZXR1cm4gdGhpcy5fb3NjUGFyc2VyLnJlZ2lzdGVySGFuZGxlcihlLHQpfSxyLnByb3RvdHlwZS5jbGVhck9zY0hhbmRsZXI9ZnVuY3Rpb24oZSl7dGhpcy5fb3NjUGFyc2VyLmNsZWFySGFuZGxlcihlKX0sci5wcm90b3R5cGUuc2V0T3NjSGFuZGxlckZhbGxiYWNrPWZ1bmN0aW9uKGUpe3RoaXMuX29zY1BhcnNlci5zZXRIYW5kbGVyRmFsbGJhY2soZSl9LHIucHJvdG90eXBlLnNldEVycm9ySGFuZGxlcj1mdW5jdGlvbihlKXt0aGlzLl9lcnJvckhhbmRsZXI9ZX0sci5wcm90b3R5cGUuY2xlYXJFcnJvckhhbmRsZXI9ZnVuY3Rpb24oKXt0aGlzLl9lcnJvckhhbmRsZXI9dGhpcy5fZXJyb3JIYW5kbGVyRmJ9LHIucHJvdG90eXBlLnJlc2V0PWZ1bmN0aW9uKCl7dGhpcy5jdXJyZW50U3RhdGU9dGhpcy5pbml0aWFsU3RhdGUsdGhpcy5fb3NjUGFyc2VyLnJlc2V0KCksdGhpcy5fZGNzUGFyc2VyLnJlc2V0KCksdGhpcy5fcGFyYW1zLnJlc2V0KCksdGhpcy5fcGFyYW1zLmFkZFBhcmFtKDApLHRoaXMuX2NvbGxlY3Q9MCx0aGlzLnByZWNlZGluZ0NvZGVwb2ludD0wLDAhPT10aGlzLl9wYXJzZVN0YWNrLnN0YXRlJiYodGhpcy5fcGFyc2VTdGFjay5zdGF0ZT0yLHRoaXMuX3BhcnNlU3RhY2suaGFuZGxlcnM9W10pfSxyLnByb3RvdHlwZS5fcHJlc2VydmVTdGFjaz1mdW5jdGlvbihlLHQscixpLG4pe3RoaXMuX3BhcnNlU3RhY2suc3RhdGU9ZSx0aGlzLl9wYXJzZVN0YWNrLmhhbmRsZXJzPXQsdGhpcy5fcGFyc2VTdGFjay5oYW5kbGVyUG9zPXIsdGhpcy5fcGFyc2VTdGFjay50cmFuc2l0aW9uPWksdGhpcy5fcGFyc2VTdGFjay5jaHVua1Bvcz1ufSxyLnByb3RvdHlwZS5wYXJzZT1mdW5jdGlvbihlLHQscil7dmFyIGksbj0wLG89MCxzPTA7aWYodGhpcy5fcGFyc2VTdGFjay5zdGF0ZSlpZigyPT09dGhpcy5fcGFyc2VTdGFjay5zdGF0ZSl0aGlzLl9wYXJzZVN0YWNrLnN0YXRlPTAscz10aGlzLl9wYXJzZVN0YWNrLmNodW5rUG9zKzE7ZWxzZXtpZih2b2lkIDA9PT1yfHwxPT09dGhpcy5fcGFyc2VTdGFjay5zdGF0ZSl0aHJvdyB0aGlzLl9wYXJzZVN0YWNrLnN0YXRlPTEsbmV3IEVycm9yKCJpbXByb3BlciBjb250aW51YXRpb24gZHVlIHRvIHByZXZpb3VzIGFzeW5jIGhhbmRsZXIsIGdpdmluZyB1cCBwYXJzaW5nIik7dmFyIGE9dGhpcy5fcGFyc2VTdGFjay5oYW5kbGVycyxjPXRoaXMuX3BhcnNlU3RhY2suaGFuZGxlclBvcy0xO3N3aXRjaCh0aGlzLl9wYXJzZVN0YWNrLnN0YXRlKXtjYXNlIDM6aWYoITE9PT1yJiZjPi0xKWZvcig7Yz49MCYmITAhPT0oaT1hW2NdKHRoaXMuX3BhcmFtcykpO2MtLSlpZihpIGluc3RhbmNlb2YgUHJvbWlzZSlyZXR1cm4gdGhpcy5fcGFyc2VTdGFjay5oYW5kbGVyUG9zPWMsaTt0aGlzLl9wYXJzZVN0YWNrLmhhbmRsZXJzPVtdO2JyZWFrO2Nhc2UgNDppZighMT09PXImJmM+LTEpZm9yKDtjPj0wJiYhMCE9PShpPWFbY10oKSk7Yy0tKWlmKGkgaW5zdGFuY2VvZiBQcm9taXNlKXJldHVybiB0aGlzLl9wYXJzZVN0YWNrLmhhbmRsZXJQb3M9YyxpO3RoaXMuX3BhcnNlU3RhY2suaGFuZGxlcnM9W107YnJlYWs7Y2FzZSA2OmlmKG49ZVt0aGlzLl9wYXJzZVN0YWNrLmNodW5rUG9zXSxpPXRoaXMuX2Rjc1BhcnNlci51bmhvb2soMjQhPT1uJiYyNiE9PW4scikpcmV0dXJuIGk7Mjc9PT1uJiYodGhpcy5fcGFyc2VTdGFjay50cmFuc2l0aW9ufD0xKSx0aGlzLl9wYXJhbXMucmVzZXQoKSx0aGlzLl9wYXJhbXMuYWRkUGFyYW0oMCksdGhpcy5fY29sbGVjdD0wO2JyZWFrO2Nhc2UgNTppZihuPWVbdGhpcy5fcGFyc2VTdGFjay5jaHVua1Bvc10saT10aGlzLl9vc2NQYXJzZXIuZW5kKDI0IT09biYmMjYhPT1uLHIpKXJldHVybiBpOzI3PT09biYmKHRoaXMuX3BhcnNlU3RhY2sudHJhbnNpdGlvbnw9MSksdGhpcy5fcGFyYW1zLnJlc2V0KCksdGhpcy5fcGFyYW1zLmFkZFBhcmFtKDApLHRoaXMuX2NvbGxlY3Q9MH10aGlzLl9wYXJzZVN0YWNrLnN0YXRlPTAscz10aGlzLl9wYXJzZVN0YWNrLmNodW5rUG9zKzEsdGhpcy5wcmVjZWRpbmdDb2RlcG9pbnQ9MCx0aGlzLmN1cnJlbnRTdGF0ZT0xNSZ0aGlzLl9wYXJzZVN0YWNrLnRyYW5zaXRpb259Zm9yKHZhciBsPXM7bDx0OysrbCl7c3dpdGNoKG49ZVtsXSwobz10aGlzLl90cmFuc2l0aW9ucy50YWJsZVt0aGlzLmN1cnJlbnRTdGF0ZTw8OHwobjwxNjA/bjpoKV0pPj40KXtjYXNlIDI6Zm9yKHZhciB1PWwrMTs7Kyt1KXtpZih1Pj10fHwobj1lW3VdKTwzMnx8bj4xMjYmJm48aCl7dGhpcy5fcHJpbnRIYW5kbGVyKGUsbCx1KSxsPXUtMTticmVha31pZigrK3U+PXR8fChuPWVbdV0pPDMyfHxuPjEyNiYmbjxoKXt0aGlzLl9wcmludEhhbmRsZXIoZSxsLHUpLGw9dS0xO2JyZWFrfWlmKCsrdT49dHx8KG49ZVt1XSk8MzJ8fG4+MTI2JiZuPGgpe3RoaXMuX3ByaW50SGFuZGxlcihlLGwsdSksbD11LTE7YnJlYWt9aWYoKyt1Pj10fHwobj1lW3VdKTwzMnx8bj4xMjYmJm48aCl7dGhpcy5fcHJpbnRIYW5kbGVyKGUsbCx1KSxsPXUtMTticmVha319YnJlYWs7Y2FzZSAzOnRoaXMuX2V4ZWN1dGVIYW5kbGVyc1tuXT90aGlzLl9leGVjdXRlSGFuZGxlcnNbbl0oKTp0aGlzLl9leGVjdXRlSGFuZGxlckZiKG4pLHRoaXMucHJlY2VkaW5nQ29kZXBvaW50PTA7YnJlYWs7Y2FzZSAwOmJyZWFrO2Nhc2UgMTppZih0aGlzLl9lcnJvckhhbmRsZXIoe3Bvc2l0aW9uOmwsY29kZTpuLGN1cnJlbnRTdGF0ZTp0aGlzLmN1cnJlbnRTdGF0ZSxjb2xsZWN0OnRoaXMuX2NvbGxlY3QscGFyYW1zOnRoaXMuX3BhcmFtcyxhYm9ydDohMX0pLmFib3J0KXJldHVybjticmVhaztjYXNlIDc6Zm9yKHZhciBmPShhPXRoaXMuX2NzaUhhbmRsZXJzW3RoaXMuX2NvbGxlY3Q8PDh8bl0pP2EubGVuZ3RoLTE6LTE7Zj49MCYmITAhPT0oaT1hW2ZdKHRoaXMuX3BhcmFtcykpO2YtLSlpZihpIGluc3RhbmNlb2YgUHJvbWlzZSlyZXR1cm4gdGhpcy5fcHJlc2VydmVTdGFjaygzLGEsZixvLGwpLGk7ZjwwJiZ0aGlzLl9jc2lIYW5kbGVyRmIodGhpcy5fY29sbGVjdDw8OHxuLHRoaXMuX3BhcmFtcyksdGhpcy5wcmVjZWRpbmdDb2RlcG9pbnQ9MDticmVhaztjYXNlIDg6ZG97c3dpdGNoKG4pe2Nhc2UgNTk6dGhpcy5fcGFyYW1zLmFkZFBhcmFtKDApO2JyZWFrO2Nhc2UgNTg6dGhpcy5fcGFyYW1zLmFkZFN1YlBhcmFtKC0xKTticmVhaztkZWZhdWx0OnRoaXMuX3BhcmFtcy5hZGREaWdpdChuLTQ4KX19d2hpbGUoKytsPHQmJihuPWVbbF0pPjQ3JiZuPDYwKTtsLS07YnJlYWs7Y2FzZSA5OnRoaXMuX2NvbGxlY3Q8PD04LHRoaXMuX2NvbGxlY3R8PW47YnJlYWs7Y2FzZSAxMDpmb3IodmFyIF89dGhpcy5fZXNjSGFuZGxlcnNbdGhpcy5fY29sbGVjdDw8OHxuXSxkPV8/Xy5sZW5ndGgtMTotMTtkPj0wJiYhMCE9PShpPV9bZF0oKSk7ZC0tKWlmKGkgaW5zdGFuY2VvZiBQcm9taXNlKXJldHVybiB0aGlzLl9wcmVzZXJ2ZVN0YWNrKDQsXyxkLG8sbCksaTtkPDAmJnRoaXMuX2VzY0hhbmRsZXJGYih0aGlzLl9jb2xsZWN0PDw4fG4pLHRoaXMucHJlY2VkaW5nQ29kZXBvaW50PTA7YnJlYWs7Y2FzZSAxMTp0aGlzLl9wYXJhbXMucmVzZXQoKSx0aGlzLl9wYXJhbXMuYWRkUGFyYW0oMCksdGhpcy5fY29sbGVjdD0wO2JyZWFrO2Nhc2UgMTI6dGhpcy5fZGNzUGFyc2VyLmhvb2sodGhpcy5fY29sbGVjdDw8OHxuLHRoaXMuX3BhcmFtcyk7YnJlYWs7Y2FzZSAxMzpmb3IodmFyIHA9bCsxOzsrK3ApaWYocD49dHx8MjQ9PT0obj1lW3BdKXx8MjY9PT1ufHwyNz09PW58fG4+MTI3JiZuPGgpe3RoaXMuX2Rjc1BhcnNlci5wdXQoZSxsLHApLGw9cC0xO2JyZWFrfWJyZWFrO2Nhc2UgMTQ6aWYoaT10aGlzLl9kY3NQYXJzZXIudW5ob29rKDI0IT09biYmMjYhPT1uKSlyZXR1cm4gdGhpcy5fcHJlc2VydmVTdGFjayg2LFtdLDAsbyxsKSxpOzI3PT09biYmKG98PTEpLHRoaXMuX3BhcmFtcy5yZXNldCgpLHRoaXMuX3BhcmFtcy5hZGRQYXJhbSgwKSx0aGlzLl9jb2xsZWN0PTAsdGhpcy5wcmVjZWRpbmdDb2RlcG9pbnQ9MDticmVhaztjYXNlIDQ6dGhpcy5fb3NjUGFyc2VyLnN0YXJ0KCk7YnJlYWs7Y2FzZSA1OmZvcih2YXIgdj1sKzE7O3YrKylpZih2Pj10fHwobj1lW3ZdKTwzMnx8bj4xMjcmJm48aCl7dGhpcy5fb3NjUGFyc2VyLnB1dChlLGwsdiksbD12LTE7YnJlYWt9YnJlYWs7Y2FzZSA2OmlmKGk9dGhpcy5fb3NjUGFyc2VyLmVuZCgyNCE9PW4mJjI2IT09bikpcmV0dXJuIHRoaXMuX3ByZXNlcnZlU3RhY2soNSxbXSwwLG8sbCksaTsyNz09PW4mJihvfD0xKSx0aGlzLl9wYXJhbXMucmVzZXQoKSx0aGlzLl9wYXJhbXMuYWRkUGFyYW0oMCksdGhpcy5fY29sbGVjdD0wLHRoaXMucHJlY2VkaW5nQ29kZXBvaW50PTB9dGhpcy5jdXJyZW50U3RhdGU9MTUmb319LHJ9KG8uRGlzcG9zYWJsZSk7dC5Fc2NhcGVTZXF1ZW5jZVBhcnNlcj1mfSw2MjQyOihlLHQscik9PntPYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC5Pc2NIYW5kbGVyPXQuT3NjUGFyc2VyPXZvaWQgMDt2YXIgaT1yKDU3NzApLG49cig0ODIpLG89W10scz1mdW5jdGlvbigpe2Z1bmN0aW9uIGUoKXt0aGlzLl9zdGF0ZT0wLHRoaXMuX2FjdGl2ZT1vLHRoaXMuX2lkPS0xLHRoaXMuX2hhbmRsZXJzPU9iamVjdC5jcmVhdGUobnVsbCksdGhpcy5faGFuZGxlckZiPWZ1bmN0aW9uKCl7fSx0aGlzLl9zdGFjaz17cGF1c2VkOiExLGxvb3BQb3NpdGlvbjowLGZhbGxUaHJvdWdoOiExfX1yZXR1cm4gZS5wcm90b3R5cGUucmVnaXN0ZXJIYW5kbGVyPWZ1bmN0aW9uKGUsdCl7dm9pZCAwPT09dGhpcy5faGFuZGxlcnNbZV0mJih0aGlzLl9oYW5kbGVyc1tlXT1bXSk7dmFyIHI9dGhpcy5faGFuZGxlcnNbZV07cmV0dXJuIHIucHVzaCh0KSx7ZGlzcG9zZTpmdW5jdGlvbigpe3ZhciBlPXIuaW5kZXhPZih0KTstMSE9PWUmJnIuc3BsaWNlKGUsMSl9fX0sZS5wcm90b3R5cGUuY2xlYXJIYW5kbGVyPWZ1bmN0aW9uKGUpe3RoaXMuX2hhbmRsZXJzW2VdJiZkZWxldGUgdGhpcy5faGFuZGxlcnNbZV19LGUucHJvdG90eXBlLnNldEhhbmRsZXJGYWxsYmFjaz1mdW5jdGlvbihlKXt0aGlzLl9oYW5kbGVyRmI9ZX0sZS5wcm90b3R5cGUuZGlzcG9zZT1mdW5jdGlvbigpe3RoaXMuX2hhbmRsZXJzPU9iamVjdC5jcmVhdGUobnVsbCksdGhpcy5faGFuZGxlckZiPWZ1bmN0aW9uKCl7fSx0aGlzLl9hY3RpdmU9b30sZS5wcm90b3R5cGUucmVzZXQ9ZnVuY3Rpb24oKXtpZigyPT09dGhpcy5fc3RhdGUpZm9yKHZhciBlPXRoaXMuX3N0YWNrLnBhdXNlZD90aGlzLl9zdGFjay5sb29wUG9zaXRpb24tMTp0aGlzLl9hY3RpdmUubGVuZ3RoLTE7ZT49MDstLWUpdGhpcy5fYWN0aXZlW2VdLmVuZCghMSk7dGhpcy5fc3RhY2sucGF1c2VkPSExLHRoaXMuX2FjdGl2ZT1vLHRoaXMuX2lkPS0xLHRoaXMuX3N0YXRlPTB9LGUucHJvdG90eXBlLl9zdGFydD1mdW5jdGlvbigpe2lmKHRoaXMuX2FjdGl2ZT10aGlzLl9oYW5kbGVyc1t0aGlzLl9pZF18fG8sdGhpcy5fYWN0aXZlLmxlbmd0aClmb3IodmFyIGU9dGhpcy5fYWN0aXZlLmxlbmd0aC0xO2U+PTA7ZS0tKXRoaXMuX2FjdGl2ZVtlXS5zdGFydCgpO2Vsc2UgdGhpcy5faGFuZGxlckZiKHRoaXMuX2lkLCJTVEFSVCIpfSxlLnByb3RvdHlwZS5fcHV0PWZ1bmN0aW9uKGUsdCxyKXtpZih0aGlzLl9hY3RpdmUubGVuZ3RoKWZvcih2YXIgaT10aGlzLl9hY3RpdmUubGVuZ3RoLTE7aT49MDtpLS0pdGhpcy5fYWN0aXZlW2ldLnB1dChlLHQscik7ZWxzZSB0aGlzLl9oYW5kbGVyRmIodGhpcy5faWQsIlBVVCIsKDAsbi51dGYzMlRvU3RyaW5nKShlLHQscikpfSxlLnByb3RvdHlwZS5zdGFydD1mdW5jdGlvbigpe3RoaXMucmVzZXQoKSx0aGlzLl9zdGF0ZT0xfSxlLnByb3RvdHlwZS5wdXQ9ZnVuY3Rpb24oZSx0LHIpe2lmKDMhPT10aGlzLl9zdGF0ZSl7aWYoMT09PXRoaXMuX3N0YXRlKWZvcig7dDxyOyl7dmFyIGk9ZVt0KytdO2lmKDU5PT09aSl7dGhpcy5fc3RhdGU9Mix0aGlzLl9zdGFydCgpO2JyZWFrfWlmKGk8NDh8fDU3PGkpcmV0dXJuIHZvaWQodGhpcy5fc3RhdGU9Myk7LTE9PT10aGlzLl9pZCYmKHRoaXMuX2lkPTApLHRoaXMuX2lkPTEwKnRoaXMuX2lkK2ktNDh9Mj09PXRoaXMuX3N0YXRlJiZyLXQ+MCYmdGhpcy5fcHV0KGUsdCxyKX19LGUucHJvdG90eXBlLmVuZD1mdW5jdGlvbihlLHQpe2lmKHZvaWQgMD09PXQmJih0PSEwKSwwIT09dGhpcy5fc3RhdGUpe2lmKDMhPT10aGlzLl9zdGF0ZSlpZigxPT09dGhpcy5fc3RhdGUmJnRoaXMuX3N0YXJ0KCksdGhpcy5fYWN0aXZlLmxlbmd0aCl7dmFyIHI9ITEsaT10aGlzLl9hY3RpdmUubGVuZ3RoLTEsbj0hMTtpZih0aGlzLl9zdGFjay5wYXVzZWQmJihpPXRoaXMuX3N0YWNrLmxvb3BQb3NpdGlvbi0xLHI9dCxuPXRoaXMuX3N0YWNrLmZhbGxUaHJvdWdoLHRoaXMuX3N0YWNrLnBhdXNlZD0hMSksIW4mJiExPT09cil7Zm9yKDtpPj0wJiYhMCE9PShyPXRoaXMuX2FjdGl2ZVtpXS5lbmQoZSkpO2ktLSlpZihyIGluc3RhbmNlb2YgUHJvbWlzZSlyZXR1cm4gdGhpcy5fc3RhY2sucGF1c2VkPSEwLHRoaXMuX3N0YWNrLmxvb3BQb3NpdGlvbj1pLHRoaXMuX3N0YWNrLmZhbGxUaHJvdWdoPSExLHI7aS0tfWZvcig7aT49MDtpLS0paWYoKHI9dGhpcy5fYWN0aXZlW2ldLmVuZCghMSkpaW5zdGFuY2VvZiBQcm9taXNlKXJldHVybiB0aGlzLl9zdGFjay5wYXVzZWQ9ITAsdGhpcy5fc3RhY2subG9vcFBvc2l0aW9uPWksdGhpcy5fc3RhY2suZmFsbFRocm91Z2g9ITAscn1lbHNlIHRoaXMuX2hhbmRsZXJGYih0aGlzLl9pZCwiRU5EIixlKTt0aGlzLl9hY3RpdmU9byx0aGlzLl9pZD0tMSx0aGlzLl9zdGF0ZT0wfX0sZX0oKTt0Lk9zY1BhcnNlcj1zO3ZhciBhPWZ1bmN0aW9uKCl7ZnVuY3Rpb24gZShlKXt0aGlzLl9oYW5kbGVyPWUsdGhpcy5fZGF0YT0iIix0aGlzLl9oaXRMaW1pdD0hMX1yZXR1cm4gZS5wcm90b3R5cGUuc3RhcnQ9ZnVuY3Rpb24oKXt0aGlzLl9kYXRhPSIiLHRoaXMuX2hpdExpbWl0PSExfSxlLnByb3RvdHlwZS5wdXQ9ZnVuY3Rpb24oZSx0LHIpe3RoaXMuX2hpdExpbWl0fHwodGhpcy5fZGF0YSs9KDAsbi51dGYzMlRvU3RyaW5nKShlLHQsciksdGhpcy5fZGF0YS5sZW5ndGg+aS5QQVlMT0FEX0xJTUlUJiYodGhpcy5fZGF0YT0iIix0aGlzLl9oaXRMaW1pdD0hMCkpfSxlLnByb3RvdHlwZS5lbmQ9ZnVuY3Rpb24oZSl7dmFyIHQ9dGhpcyxyPSExO2lmKHRoaXMuX2hpdExpbWl0KXI9ITE7ZWxzZSBpZihlJiYocj10aGlzLl9oYW5kbGVyKHRoaXMuX2RhdGEpKWluc3RhbmNlb2YgUHJvbWlzZSlyZXR1cm4gci50aGVuKChmdW5jdGlvbihlKXtyZXR1cm4gdC5fZGF0YT0iIix0Ll9oaXRMaW1pdD0hMSxlfSkpO3JldHVybiB0aGlzLl9kYXRhPSIiLHRoaXMuX2hpdExpbWl0PSExLHJ9LGV9KCk7dC5Pc2NIYW5kbGVyPWF9LDg3NDI6KGUsdCk9PntPYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC5QYXJhbXM9dm9pZCAwO3ZhciByPTIxNDc0ODM2NDcsaT1mdW5jdGlvbigpe2Z1bmN0aW9uIGUoZSx0KXtpZih2b2lkIDA9PT1lJiYoZT0zMiksdm9pZCAwPT09dCYmKHQ9MzIpLHRoaXMubWF4TGVuZ3RoPWUsdGhpcy5tYXhTdWJQYXJhbXNMZW5ndGg9dCx0PjI1Nil0aHJvdyBuZXcgRXJyb3IoIm1heFN1YlBhcmFtc0xlbmd0aCBtdXN0IG5vdCBiZSBncmVhdGVyIHRoYW4gMjU2Iik7dGhpcy5wYXJhbXM9bmV3IEludDMyQXJyYXkoZSksdGhpcy5sZW5ndGg9MCx0aGlzLl9zdWJQYXJhbXM9bmV3IEludDMyQXJyYXkodCksdGhpcy5fc3ViUGFyYW1zTGVuZ3RoPTAsdGhpcy5fc3ViUGFyYW1zSWR4PW5ldyBVaW50MTZBcnJheShlKSx0aGlzLl9yZWplY3REaWdpdHM9ITEsdGhpcy5fcmVqZWN0U3ViRGlnaXRzPSExLHRoaXMuX2RpZ2l0SXNTdWI9ITF9cmV0dXJuIGUuZnJvbUFycmF5PWZ1bmN0aW9uKHQpe3ZhciByPW5ldyBlO2lmKCF0Lmxlbmd0aClyZXR1cm4gcjtmb3IodmFyIGk9QXJyYXkuaXNBcnJheSh0WzBdKT8xOjA7aTx0Lmxlbmd0aDsrK2kpe3ZhciBuPXRbaV07aWYoQXJyYXkuaXNBcnJheShuKSlmb3IodmFyIG89MDtvPG4ubGVuZ3RoOysrbylyLmFkZFN1YlBhcmFtKG5bb10pO2Vsc2Ugci5hZGRQYXJhbShuKX1yZXR1cm4gcn0sZS5wcm90b3R5cGUuY2xvbmU9ZnVuY3Rpb24oKXt2YXIgdD1uZXcgZSh0aGlzLm1heExlbmd0aCx0aGlzLm1heFN1YlBhcmFtc0xlbmd0aCk7cmV0dXJuIHQucGFyYW1zLnNldCh0aGlzLnBhcmFtcyksdC5sZW5ndGg9dGhpcy5sZW5ndGgsdC5fc3ViUGFyYW1zLnNldCh0aGlzLl9zdWJQYXJhbXMpLHQuX3N1YlBhcmFtc0xlbmd0aD10aGlzLl9zdWJQYXJhbXNMZW5ndGgsdC5fc3ViUGFyYW1zSWR4LnNldCh0aGlzLl9zdWJQYXJhbXNJZHgpLHQuX3JlamVjdERpZ2l0cz10aGlzLl9yZWplY3REaWdpdHMsdC5fcmVqZWN0U3ViRGlnaXRzPXRoaXMuX3JlamVjdFN1YkRpZ2l0cyx0Ll9kaWdpdElzU3ViPXRoaXMuX2RpZ2l0SXNTdWIsdH0sZS5wcm90b3R5cGUudG9BcnJheT1mdW5jdGlvbigpe2Zvcih2YXIgZT1bXSx0PTA7dDx0aGlzLmxlbmd0aDsrK3Qpe2UucHVzaCh0aGlzLnBhcmFtc1t0XSk7dmFyIHI9dGhpcy5fc3ViUGFyYW1zSWR4W3RdPj44LGk9MjU1JnRoaXMuX3N1YlBhcmFtc0lkeFt0XTtpLXI+MCYmZS5wdXNoKEFycmF5LnByb3RvdHlwZS5zbGljZS5jYWxsKHRoaXMuX3N1YlBhcmFtcyxyLGkpKX1yZXR1cm4gZX0sZS5wcm90b3R5cGUucmVzZXQ9ZnVuY3Rpb24oKXt0aGlzLmxlbmd0aD0wLHRoaXMuX3N1YlBhcmFtc0xlbmd0aD0wLHRoaXMuX3JlamVjdERpZ2l0cz0hMSx0aGlzLl9yZWplY3RTdWJEaWdpdHM9ITEsdGhpcy5fZGlnaXRJc1N1Yj0hMX0sZS5wcm90b3R5cGUuYWRkUGFyYW09ZnVuY3Rpb24oZSl7aWYodGhpcy5fZGlnaXRJc1N1Yj0hMSx0aGlzLmxlbmd0aD49dGhpcy5tYXhMZW5ndGgpdGhpcy5fcmVqZWN0RGlnaXRzPSEwO2Vsc2V7aWYoZTwtMSl0aHJvdyBuZXcgRXJyb3IoInZhbHVlcyBsZXNzZXIgdGhhbiAtMSBhcmUgbm90IGFsbG93ZWQiKTt0aGlzLl9zdWJQYXJhbXNJZHhbdGhpcy5sZW5ndGhdPXRoaXMuX3N1YlBhcmFtc0xlbmd0aDw8OHx0aGlzLl9zdWJQYXJhbXNMZW5ndGgsdGhpcy5wYXJhbXNbdGhpcy5sZW5ndGgrK109ZT5yP3I6ZX19LGUucHJvdG90eXBlLmFkZFN1YlBhcmFtPWZ1bmN0aW9uKGUpe2lmKHRoaXMuX2RpZ2l0SXNTdWI9ITAsdGhpcy5sZW5ndGgpaWYodGhpcy5fcmVqZWN0RGlnaXRzfHx0aGlzLl9zdWJQYXJhbXNMZW5ndGg+PXRoaXMubWF4U3ViUGFyYW1zTGVuZ3RoKXRoaXMuX3JlamVjdFN1YkRpZ2l0cz0hMDtlbHNle2lmKGU8LTEpdGhyb3cgbmV3IEVycm9yKCJ2YWx1ZXMgbGVzc2VyIHRoYW4gLTEgYXJlIG5vdCBhbGxvd2VkIik7dGhpcy5fc3ViUGFyYW1zW3RoaXMuX3N1YlBhcmFtc0xlbmd0aCsrXT1lPnI/cjplLHRoaXMuX3N1YlBhcmFtc0lkeFt0aGlzLmxlbmd0aC0xXSsrfX0sZS5wcm90b3R5cGUuaGFzU3ViUGFyYW1zPWZ1bmN0aW9uKGUpe3JldHVybigyNTUmdGhpcy5fc3ViUGFyYW1zSWR4W2VdKS0odGhpcy5fc3ViUGFyYW1zSWR4W2VdPj44KT4wfSxlLnByb3RvdHlwZS5nZXRTdWJQYXJhbXM9ZnVuY3Rpb24oZSl7dmFyIHQ9dGhpcy5fc3ViUGFyYW1zSWR4W2VdPj44LHI9MjU1JnRoaXMuX3N1YlBhcmFtc0lkeFtlXTtyZXR1cm4gci10PjA/dGhpcy5fc3ViUGFyYW1zLnN1YmFycmF5KHQscik6bnVsbH0sZS5wcm90b3R5cGUuZ2V0U3ViUGFyYW1zQWxsPWZ1bmN0aW9uKCl7Zm9yKHZhciBlPXt9LHQ9MDt0PHRoaXMubGVuZ3RoOysrdCl7dmFyIHI9dGhpcy5fc3ViUGFyYW1zSWR4W3RdPj44LGk9MjU1JnRoaXMuX3N1YlBhcmFtc0lkeFt0XTtpLXI+MCYmKGVbdF09dGhpcy5fc3ViUGFyYW1zLnNsaWNlKHIsaSkpfXJldHVybiBlfSxlLnByb3RvdHlwZS5hZGREaWdpdD1mdW5jdGlvbihlKXt2YXIgdDtpZighKHRoaXMuX3JlamVjdERpZ2l0c3x8ISh0PXRoaXMuX2RpZ2l0SXNTdWI/dGhpcy5fc3ViUGFyYW1zTGVuZ3RoOnRoaXMubGVuZ3RoKXx8dGhpcy5fZGlnaXRJc1N1YiYmdGhpcy5fcmVqZWN0U3ViRGlnaXRzKSl7dmFyIGk9dGhpcy5fZGlnaXRJc1N1Yj90aGlzLl9zdWJQYXJhbXM6dGhpcy5wYXJhbXMsbj1pW3QtMV07aVt0LTFdPX5uP01hdGgubWluKDEwKm4rZSxyKTplfX0sZX0oKTt0LlBhcmFtcz1pfSw1NzQxOihlLHQpPT57T2JqZWN0LmRlZmluZVByb3BlcnR5KHQsIl9fZXNNb2R1bGUiLHt2YWx1ZTohMH0pLHQuQWRkb25NYW5hZ2VyPXZvaWQgMDt2YXIgcj1mdW5jdGlvbigpe2Z1bmN0aW9uIGUoKXt0aGlzLl9hZGRvbnM9W119cmV0dXJuIGUucHJvdG90eXBlLmRpc3Bvc2U9ZnVuY3Rpb24oKXtmb3IodmFyIGU9dGhpcy5fYWRkb25zLmxlbmd0aC0xO2U+PTA7ZS0tKXRoaXMuX2FkZG9uc1tlXS5pbnN0YW5jZS5kaXNwb3NlKCl9LGUucHJvdG90eXBlLmxvYWRBZGRvbj1mdW5jdGlvbihlLHQpe3ZhciByPXRoaXMsaT17aW5zdGFuY2U6dCxkaXNwb3NlOnQuZGlzcG9zZSxpc0Rpc3Bvc2VkOiExfTt0aGlzLl9hZGRvbnMucHVzaChpKSx0LmRpc3Bvc2U9ZnVuY3Rpb24oKXtyZXR1cm4gci5fd3JhcHBlZEFkZG9uRGlzcG9zZShpKX0sdC5hY3RpdmF0ZShlKX0sZS5wcm90b3R5cGUuX3dyYXBwZWRBZGRvbkRpc3Bvc2U9ZnVuY3Rpb24oZSl7aWYoIWUuaXNEaXNwb3NlZCl7Zm9yKHZhciB0PS0xLHI9MDtyPHRoaXMuX2FkZG9ucy5sZW5ndGg7cisrKWlmKHRoaXMuX2FkZG9uc1tyXT09PWUpe3Q9cjticmVha31pZigtMT09PXQpdGhyb3cgbmV3IEVycm9yKCJDb3VsZCBub3QgZGlzcG9zZSBhbiBhZGRvbiB0aGF0IGhhcyBub3QgYmVlbiBsb2FkZWQiKTtlLmlzRGlzcG9zZWQ9ITAsZS5kaXNwb3NlLmFwcGx5KGUuaW5zdGFuY2UpLHRoaXMuX2FkZG9ucy5zcGxpY2UodCwxKX19LGV9KCk7dC5BZGRvbk1hbmFnZXI9cn0sODc3MTooZSx0LHIpPT57T2JqZWN0LmRlZmluZVByb3BlcnR5KHQsIl9fZXNNb2R1bGUiLHt2YWx1ZTohMH0pLHQuQnVmZmVyQXBpVmlldz12b2lkIDA7dmFyIGk9cigzNzg1KSxuPXIoNTExKSxvPWZ1bmN0aW9uKCl7ZnVuY3Rpb24gZShlLHQpe3RoaXMuX2J1ZmZlcj1lLHRoaXMudHlwZT10fXJldHVybiBlLnByb3RvdHlwZS5pbml0PWZ1bmN0aW9uKGUpe3JldHVybiB0aGlzLl9idWZmZXI9ZSx0aGlzfSxPYmplY3QuZGVmaW5lUHJvcGVydHkoZS5wcm90b3R5cGUsImN1cnNvclkiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fYnVmZmVyLnl9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KGUucHJvdG90eXBlLCJjdXJzb3JYIix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX2J1ZmZlci54fSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLE9iamVjdC5kZWZpbmVQcm9wZXJ0eShlLnByb3RvdHlwZSwidmlld3BvcnRZIix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX2J1ZmZlci55ZGlzcH0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxPYmplY3QuZGVmaW5lUHJvcGVydHkoZS5wcm90b3R5cGUsImJhc2VZIix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX2J1ZmZlci55YmFzZX0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxPYmplY3QuZGVmaW5lUHJvcGVydHkoZS5wcm90b3R5cGUsImxlbmd0aCIse2dldDpmdW5jdGlvbigpe3JldHVybiB0aGlzLl9idWZmZXIubGluZXMubGVuZ3RofSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLGUucHJvdG90eXBlLmdldExpbmU9ZnVuY3Rpb24oZSl7dmFyIHQ9dGhpcy5fYnVmZmVyLmxpbmVzLmdldChlKTtpZih0KXJldHVybiBuZXcgaS5CdWZmZXJMaW5lQXBpVmlldyh0KX0sZS5wcm90b3R5cGUuZ2V0TnVsbENlbGw9ZnVuY3Rpb24oKXtyZXR1cm4gbmV3IG4uQ2VsbERhdGF9LGV9KCk7dC5CdWZmZXJBcGlWaWV3PW99LDM3ODU6KGUsdCxyKT0+e09iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LCJfX2VzTW9kdWxlIix7dmFsdWU6ITB9KSx0LkJ1ZmZlckxpbmVBcGlWaWV3PXZvaWQgMDt2YXIgaT1yKDUxMSksbj1mdW5jdGlvbigpe2Z1bmN0aW9uIGUoZSl7dGhpcy5fbGluZT1lfXJldHVybiBPYmplY3QuZGVmaW5lUHJvcGVydHkoZS5wcm90b3R5cGUsImlzV3JhcHBlZCIse2dldDpmdW5jdGlvbigpe3JldHVybiB0aGlzLl9saW5lLmlzV3JhcHBlZH0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxPYmplY3QuZGVmaW5lUHJvcGVydHkoZS5wcm90b3R5cGUsImxlbmd0aCIse2dldDpmdW5jdGlvbigpe3JldHVybiB0aGlzLl9saW5lLmxlbmd0aH0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxlLnByb3RvdHlwZS5nZXRDZWxsPWZ1bmN0aW9uKGUsdCl7aWYoIShlPDB8fGU+PXRoaXMuX2xpbmUubGVuZ3RoKSlyZXR1cm4gdD8odGhpcy5fbGluZS5sb2FkQ2VsbChlLHQpLHQpOnRoaXMuX2xpbmUubG9hZENlbGwoZSxuZXcgaS5DZWxsRGF0YSl9LGUucHJvdG90eXBlLnRyYW5zbGF0ZVRvU3RyaW5nPWZ1bmN0aW9uKGUsdCxyKXtyZXR1cm4gdGhpcy5fbGluZS50cmFuc2xhdGVUb1N0cmluZyhlLHQscil9LGV9KCk7dC5CdWZmZXJMaW5lQXBpVmlldz1ufSw4Mjg1OihlLHQscik9PntPYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC5CdWZmZXJOYW1lc3BhY2VBcGk9dm9pZCAwO3ZhciBpPXIoODc3MSksbj1yKDg0NjApLG89ZnVuY3Rpb24oKXtmdW5jdGlvbiBlKGUpe3ZhciB0PXRoaXM7dGhpcy5fY29yZT1lLHRoaXMuX29uQnVmZmVyQ2hhbmdlPW5ldyBuLkV2ZW50RW1pdHRlcix0aGlzLl9ub3JtYWw9bmV3IGkuQnVmZmVyQXBpVmlldyh0aGlzLl9jb3JlLmJ1ZmZlcnMubm9ybWFsLCJub3JtYWwiKSx0aGlzLl9hbHRlcm5hdGU9bmV3IGkuQnVmZmVyQXBpVmlldyh0aGlzLl9jb3JlLmJ1ZmZlcnMuYWx0LCJhbHRlcm5hdGUiKSx0aGlzLl9jb3JlLmJ1ZmZlcnMub25CdWZmZXJBY3RpdmF0ZSgoZnVuY3Rpb24oKXtyZXR1cm4gdC5fb25CdWZmZXJDaGFuZ2UuZmlyZSh0LmFjdGl2ZSl9KSl9cmV0dXJuIE9iamVjdC5kZWZpbmVQcm9wZXJ0eShlLnByb3RvdHlwZSwib25CdWZmZXJDaGFuZ2UiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fb25CdWZmZXJDaGFuZ2UuZXZlbnR9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KGUucHJvdG90eXBlLCJhY3RpdmUiLHtnZXQ6ZnVuY3Rpb24oKXtpZih0aGlzLl9jb3JlLmJ1ZmZlcnMuYWN0aXZlPT09dGhpcy5fY29yZS5idWZmZXJzLm5vcm1hbClyZXR1cm4gdGhpcy5ub3JtYWw7aWYodGhpcy5fY29yZS5idWZmZXJzLmFjdGl2ZT09PXRoaXMuX2NvcmUuYnVmZmVycy5hbHQpcmV0dXJuIHRoaXMuYWx0ZXJuYXRlO3Rocm93IG5ldyBFcnJvcigiQWN0aXZlIGJ1ZmZlciBpcyBuZWl0aGVyIG5vcm1hbCBub3IgYWx0ZXJuYXRlIil9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KGUucHJvdG90eXBlLCJub3JtYWwiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fbm9ybWFsLmluaXQodGhpcy5fY29yZS5idWZmZXJzLm5vcm1hbCl9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KGUucHJvdG90eXBlLCJhbHRlcm5hdGUiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fYWx0ZXJuYXRlLmluaXQodGhpcy5fY29yZS5idWZmZXJzLmFsdCl9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksZX0oKTt0LkJ1ZmZlck5hbWVzcGFjZUFwaT1vfSw3OTc1OihlLHQpPT57T2JqZWN0LmRlZmluZVByb3BlcnR5KHQsIl9fZXNNb2R1bGUiLHt2YWx1ZTohMH0pLHQuUGFyc2VyQXBpPXZvaWQgMDt2YXIgcj1mdW5jdGlvbigpe2Z1bmN0aW9uIGUoZSl7dGhpcy5fY29yZT1lfXJldHVybiBlLnByb3RvdHlwZS5yZWdpc3RlckNzaUhhbmRsZXI9ZnVuY3Rpb24oZSx0KXtyZXR1cm4gdGhpcy5fY29yZS5yZWdpc3RlckNzaUhhbmRsZXIoZSwoZnVuY3Rpb24oZSl7cmV0dXJuIHQoZS50b0FycmF5KCkpfSkpfSxlLnByb3RvdHlwZS5hZGRDc2lIYW5kbGVyPWZ1bmN0aW9uKGUsdCl7cmV0dXJuIHRoaXMucmVnaXN0ZXJDc2lIYW5kbGVyKGUsdCl9LGUucHJvdG90eXBlLnJlZ2lzdGVyRGNzSGFuZGxlcj1mdW5jdGlvbihlLHQpe3JldHVybiB0aGlzLl9jb3JlLnJlZ2lzdGVyRGNzSGFuZGxlcihlLChmdW5jdGlvbihlLHIpe3JldHVybiB0KGUsci50b0FycmF5KCkpfSkpfSxlLnByb3RvdHlwZS5hZGREY3NIYW5kbGVyPWZ1bmN0aW9uKGUsdCl7cmV0dXJuIHRoaXMucmVnaXN0ZXJEY3NIYW5kbGVyKGUsdCl9LGUucHJvdG90eXBlLnJlZ2lzdGVyRXNjSGFuZGxlcj1mdW5jdGlvbihlLHQpe3JldHVybiB0aGlzLl9jb3JlLnJlZ2lzdGVyRXNjSGFuZGxlcihlLHQpfSxlLnByb3RvdHlwZS5hZGRFc2NIYW5kbGVyPWZ1bmN0aW9uKGUsdCl7cmV0dXJuIHRoaXMucmVnaXN0ZXJFc2NIYW5kbGVyKGUsdCl9LGUucHJvdG90eXBlLnJlZ2lzdGVyT3NjSGFuZGxlcj1mdW5jdGlvbihlLHQpe3JldHVybiB0aGlzLl9jb3JlLnJlZ2lzdGVyT3NjSGFuZGxlcihlLHQpfSxlLnByb3RvdHlwZS5hZGRPc2NIYW5kbGVyPWZ1bmN0aW9uKGUsdCl7cmV0dXJuIHRoaXMucmVnaXN0ZXJPc2NIYW5kbGVyKGUsdCl9LGV9KCk7dC5QYXJzZXJBcGk9cn0sNzA5MDooZSx0KT0+e09iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LCJfX2VzTW9kdWxlIix7dmFsdWU6ITB9KSx0LlVuaWNvZGVBcGk9dm9pZCAwO3ZhciByPWZ1bmN0aW9uKCl7ZnVuY3Rpb24gZShlKXt0aGlzLl9jb3JlPWV9cmV0dXJuIGUucHJvdG90eXBlLnJlZ2lzdGVyPWZ1bmN0aW9uKGUpe3RoaXMuX2NvcmUudW5pY29kZVNlcnZpY2UucmVnaXN0ZXIoZSl9LE9iamVjdC5kZWZpbmVQcm9wZXJ0eShlLnByb3RvdHlwZSwidmVyc2lvbnMiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fY29yZS51bmljb2RlU2VydmljZS52ZXJzaW9uc30sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxPYmplY3QuZGVmaW5lUHJvcGVydHkoZS5wcm90b3R5cGUsImFjdGl2ZVZlcnNpb24iLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fY29yZS51bmljb2RlU2VydmljZS5hY3RpdmVWZXJzaW9ufSxzZXQ6ZnVuY3Rpb24oZSl7dGhpcy5fY29yZS51bmljb2RlU2VydmljZS5hY3RpdmVWZXJzaW9uPWV9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksZX0oKTt0LlVuaWNvZGVBcGk9cn0sNzQ0OmZ1bmN0aW9uKGUsdCxyKXt2YXIgaSxuPXRoaXMmJnRoaXMuX19leHRlbmRzfHwoaT1mdW5jdGlvbihlLHQpe3JldHVybiBpPU9iamVjdC5zZXRQcm90b3R5cGVPZnx8e19fcHJvdG9fXzpbXX1pbnN0YW5jZW9mIEFycmF5JiZmdW5jdGlvbihlLHQpe2UuX19wcm90b19fPXR9fHxmdW5jdGlvbihlLHQpe2Zvcih2YXIgciBpbiB0KU9iamVjdC5wcm90b3R5cGUuaGFzT3duUHJvcGVydHkuY2FsbCh0LHIpJiYoZVtyXT10W3JdKX0saShlLHQpfSxmdW5jdGlvbihlLHQpe2lmKCJmdW5jdGlvbiIhPXR5cGVvZiB0JiZudWxsIT09dCl0aHJvdyBuZXcgVHlwZUVycm9yKCJDbGFzcyBleHRlbmRzIHZhbHVlICIrU3RyaW5nKHQpKyIgaXMgbm90IGEgY29uc3RydWN0b3Igb3IgbnVsbCIpO2Z1bmN0aW9uIHIoKXt0aGlzLmNvbnN0cnVjdG9yPWV9aShlLHQpLGUucHJvdG90eXBlPW51bGw9PT10P09iamVjdC5jcmVhdGUodCk6KHIucHJvdG90eXBlPXQucHJvdG90eXBlLG5ldyByKX0pLG89dGhpcyYmdGhpcy5fX2RlY29yYXRlfHxmdW5jdGlvbihlLHQscixpKXt2YXIgbixvPWFyZ3VtZW50cy5sZW5ndGgscz1vPDM/dDpudWxsPT09aT9pPU9iamVjdC5nZXRPd25Qcm9wZXJ0eURlc2NyaXB0b3IodCxyKTppO2lmKCJvYmplY3QiPT10eXBlb2YgUmVmbGVjdCYmImZ1bmN0aW9uIj09dHlwZW9mIFJlZmxlY3QuZGVjb3JhdGUpcz1SZWZsZWN0LmRlY29yYXRlKGUsdCxyLGkpO2Vsc2UgZm9yKHZhciBhPWUubGVuZ3RoLTE7YT49MDthLS0pKG49ZVthXSkmJihzPShvPDM/bihzKTpvPjM/bih0LHIscyk6bih0LHIpKXx8cyk7cmV0dXJuIG8+MyYmcyYmT2JqZWN0LmRlZmluZVByb3BlcnR5KHQscixzKSxzfSxzPXRoaXMmJnRoaXMuX19wYXJhbXx8ZnVuY3Rpb24oZSx0KXtyZXR1cm4gZnVuY3Rpb24ocixpKXt0KHIsaSxlKX19O09iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LCJfX2VzTW9kdWxlIix7dmFsdWU6ITB9KSx0LkJ1ZmZlclNlcnZpY2U9dC5NSU5JTVVNX1JPV1M9dC5NSU5JTVVNX0NPTFM9dm9pZCAwO3ZhciBhPXIoMjU4NSksYz1yKDUyOTUpLGw9cig4NDYwKSx1PXIoODQ0KTt0Lk1JTklNVU1fQ09MUz0yLHQuTUlOSU1VTV9ST1dTPTE7dmFyIGg9ZnVuY3Rpb24oZSl7ZnVuY3Rpb24gcihyKXt2YXIgaT1lLmNhbGwodGhpcyl8fHRoaXM7cmV0dXJuIGkuX29wdGlvbnNTZXJ2aWNlPXIsaS5pc1VzZXJTY3JvbGxpbmc9ITEsaS5fb25SZXNpemU9bmV3IGwuRXZlbnRFbWl0dGVyLGkuX29uU2Nyb2xsPW5ldyBsLkV2ZW50RW1pdHRlcixpLmNvbHM9TWF0aC5tYXgoci5vcHRpb25zLmNvbHN8fDAsdC5NSU5JTVVNX0NPTFMpLGkucm93cz1NYXRoLm1heChyLm9wdGlvbnMucm93c3x8MCx0Lk1JTklNVU1fUk9XUyksaS5idWZmZXJzPW5ldyBjLkJ1ZmZlclNldChyLGkpLGl9cmV0dXJuIG4ocixlKSxPYmplY3QuZGVmaW5lUHJvcGVydHkoci5wcm90b3R5cGUsIm9uUmVzaXplIix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX29uUmVzaXplLmV2ZW50fSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLE9iamVjdC5kZWZpbmVQcm9wZXJ0eShyLnByb3RvdHlwZSwib25TY3JvbGwiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fb25TY3JvbGwuZXZlbnR9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KHIucHJvdG90eXBlLCJidWZmZXIiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5idWZmZXJzLmFjdGl2ZX0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxyLnByb3RvdHlwZS5kaXNwb3NlPWZ1bmN0aW9uKCl7ZS5wcm90b3R5cGUuZGlzcG9zZS5jYWxsKHRoaXMpLHRoaXMuYnVmZmVycy5kaXNwb3NlKCl9LHIucHJvdG90eXBlLnJlc2l6ZT1mdW5jdGlvbihlLHQpe3RoaXMuY29scz1lLHRoaXMucm93cz10LHRoaXMuYnVmZmVycy5yZXNpemUoZSx0KSx0aGlzLmJ1ZmZlcnMuc2V0dXBUYWJTdG9wcyh0aGlzLmNvbHMpLHRoaXMuX29uUmVzaXplLmZpcmUoe2NvbHM6ZSxyb3dzOnR9KX0sci5wcm90b3R5cGUucmVzZXQ9ZnVuY3Rpb24oKXt0aGlzLmJ1ZmZlcnMucmVzZXQoKSx0aGlzLmlzVXNlclNjcm9sbGluZz0hMX0sci5wcm90b3R5cGUuc2Nyb2xsPWZ1bmN0aW9uKGUsdCl7dm9pZCAwPT09dCYmKHQ9ITEpO3ZhciByLGk9dGhpcy5idWZmZXI7KHI9dGhpcy5fY2FjaGVkQmxhbmtMaW5lKSYmci5sZW5ndGg9PT10aGlzLmNvbHMmJnIuZ2V0RmcoMCk9PT1lLmZnJiZyLmdldEJnKDApPT09ZS5iZ3x8KHI9aS5nZXRCbGFua0xpbmUoZSx0KSx0aGlzLl9jYWNoZWRCbGFua0xpbmU9ciksci5pc1dyYXBwZWQ9dDt2YXIgbj1pLnliYXNlK2kuc2Nyb2xsVG9wLG89aS55YmFzZStpLnNjcm9sbEJvdHRvbTtpZigwPT09aS5zY3JvbGxUb3Ape3ZhciBzPWkubGluZXMuaXNGdWxsO289PT1pLmxpbmVzLmxlbmd0aC0xP3M/aS5saW5lcy5yZWN5Y2xlKCkuY29weUZyb20ocik6aS5saW5lcy5wdXNoKHIuY2xvbmUoKSk6aS5saW5lcy5zcGxpY2UobysxLDAsci5jbG9uZSgpKSxzP3RoaXMuaXNVc2VyU2Nyb2xsaW5nJiYoaS55ZGlzcD1NYXRoLm1heChpLnlkaXNwLTEsMCkpOihpLnliYXNlKyssdGhpcy5pc1VzZXJTY3JvbGxpbmd8fGkueWRpc3ArKyl9ZWxzZXt2YXIgYT1vLW4rMTtpLmxpbmVzLnNoaWZ0RWxlbWVudHMobisxLGEtMSwtMSksaS5saW5lcy5zZXQobyxyLmNsb25lKCkpfXRoaXMuaXNVc2VyU2Nyb2xsaW5nfHwoaS55ZGlzcD1pLnliYXNlKSx0aGlzLl9vblNjcm9sbC5maXJlKGkueWRpc3ApfSxyLnByb3RvdHlwZS5zY3JvbGxMaW5lcz1mdW5jdGlvbihlLHQscil7dmFyIGk9dGhpcy5idWZmZXI7aWYoZTwwKXtpZigwPT09aS55ZGlzcClyZXR1cm47dGhpcy5pc1VzZXJTY3JvbGxpbmc9ITB9ZWxzZSBlK2kueWRpc3A+PWkueWJhc2UmJih0aGlzLmlzVXNlclNjcm9sbGluZz0hMSk7dmFyIG49aS55ZGlzcDtpLnlkaXNwPU1hdGgubWF4KE1hdGgubWluKGkueWRpc3ArZSxpLnliYXNlKSwwKSxuIT09aS55ZGlzcCYmKHR8fHRoaXMuX29uU2Nyb2xsLmZpcmUoaS55ZGlzcCkpfSxyLnByb3RvdHlwZS5zY3JvbGxQYWdlcz1mdW5jdGlvbihlKXt0aGlzLnNjcm9sbExpbmVzKGUqKHRoaXMucm93cy0xKSl9LHIucHJvdG90eXBlLnNjcm9sbFRvVG9wPWZ1bmN0aW9uKCl7dGhpcy5zY3JvbGxMaW5lcygtdGhpcy5idWZmZXIueWRpc3ApfSxyLnByb3RvdHlwZS5zY3JvbGxUb0JvdHRvbT1mdW5jdGlvbigpe3RoaXMuc2Nyb2xsTGluZXModGhpcy5idWZmZXIueWJhc2UtdGhpcy5idWZmZXIueWRpc3ApfSxyLnByb3RvdHlwZS5zY3JvbGxUb0xpbmU9ZnVuY3Rpb24oZSl7dmFyIHQ9ZS10aGlzLmJ1ZmZlci55ZGlzcDswIT09dCYmdGhpcy5zY3JvbGxMaW5lcyh0KX0sbyhbcygwLGEuSU9wdGlvbnNTZXJ2aWNlKV0scil9KHUuRGlzcG9zYWJsZSk7dC5CdWZmZXJTZXJ2aWNlPWh9LDc5OTQ6KGUsdCk9PntPYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC5DaGFyc2V0U2VydmljZT12b2lkIDA7dmFyIHI9ZnVuY3Rpb24oKXtmdW5jdGlvbiBlKCl7dGhpcy5nbGV2ZWw9MCx0aGlzLl9jaGFyc2V0cz1bXX1yZXR1cm4gZS5wcm90b3R5cGUucmVzZXQ9ZnVuY3Rpb24oKXt0aGlzLmNoYXJzZXQ9dm9pZCAwLHRoaXMuX2NoYXJzZXRzPVtdLHRoaXMuZ2xldmVsPTB9LGUucHJvdG90eXBlLnNldGdMZXZlbD1mdW5jdGlvbihlKXt0aGlzLmdsZXZlbD1lLHRoaXMuY2hhcnNldD10aGlzLl9jaGFyc2V0c1tlXX0sZS5wcm90b3R5cGUuc2V0Z0NoYXJzZXQ9ZnVuY3Rpb24oZSx0KXt0aGlzLl9jaGFyc2V0c1tlXT10LHRoaXMuZ2xldmVsPT09ZSYmKHRoaXMuY2hhcnNldD10KX0sZX0oKTt0LkNoYXJzZXRTZXJ2aWNlPXJ9LDE3NTM6ZnVuY3Rpb24oZSx0LHIpe3ZhciBpPXRoaXMmJnRoaXMuX19kZWNvcmF0ZXx8ZnVuY3Rpb24oZSx0LHIsaSl7dmFyIG4sbz1hcmd1bWVudHMubGVuZ3RoLHM9bzwzP3Q6bnVsbD09PWk/aT1PYmplY3QuZ2V0T3duUHJvcGVydHlEZXNjcmlwdG9yKHQscik6aTtpZigib2JqZWN0Ij09dHlwZW9mIFJlZmxlY3QmJiJmdW5jdGlvbiI9PXR5cGVvZiBSZWZsZWN0LmRlY29yYXRlKXM9UmVmbGVjdC5kZWNvcmF0ZShlLHQscixpKTtlbHNlIGZvcih2YXIgYT1lLmxlbmd0aC0xO2E+PTA7YS0tKShuPWVbYV0pJiYocz0obzwzP24ocyk6bz4zP24odCxyLHMpOm4odCxyKSl8fHMpO3JldHVybiBvPjMmJnMmJk9iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LHIscyksc30sbj10aGlzJiZ0aGlzLl9fcGFyYW18fGZ1bmN0aW9uKGUsdCl7cmV0dXJuIGZ1bmN0aW9uKHIsaSl7dChyLGksZSl9fTtPYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC5Db3JlTW91c2VTZXJ2aWNlPXZvaWQgMDt2YXIgbz1yKDI1ODUpLHM9cig4NDYwKSxhPXtOT05FOntldmVudHM6MCxyZXN0cmljdDpmdW5jdGlvbigpe3JldHVybiExfX0sWDEwOntldmVudHM6MSxyZXN0cmljdDpmdW5jdGlvbihlKXtyZXR1cm4gNCE9PWUuYnV0dG9uJiYxPT09ZS5hY3Rpb24mJihlLmN0cmw9ITEsZS5hbHQ9ITEsZS5zaGlmdD0hMSwhMCl9fSxWVDIwMDp7ZXZlbnRzOjE5LHJlc3RyaWN0OmZ1bmN0aW9uKGUpe3JldHVybiAzMiE9PWUuYWN0aW9ufX0sRFJBRzp7ZXZlbnRzOjIzLHJlc3RyaWN0OmZ1bmN0aW9uKGUpe3JldHVybiAzMiE9PWUuYWN0aW9ufHwzIT09ZS5idXR0b259fSxBTlk6e2V2ZW50czozMSxyZXN0cmljdDpmdW5jdGlvbihlKXtyZXR1cm4hMH19fTtmdW5jdGlvbiBjKGUsdCl7dmFyIHI9KGUuY3RybD8xNjowKXwoZS5zaGlmdD80OjApfChlLmFsdD84OjApO3JldHVybiA0PT09ZS5idXR0b24/KHJ8PTY0LHJ8PWUuYWN0aW9uKToocnw9MyZlLmJ1dHRvbiw0JmUuYnV0dG9uJiYocnw9NjQpLDgmZS5idXR0b24mJihyfD0xMjgpLDMyPT09ZS5hY3Rpb24/cnw9MzI6MCE9PWUuYWN0aW9ufHx0fHwocnw9MykpLHJ9dmFyIGw9U3RyaW5nLmZyb21DaGFyQ29kZSx1PXtERUZBVUxUOmZ1bmN0aW9uKGUpe3ZhciB0PVtjKGUsITEpKzMyLGUuY29sKzMyLGUucm93KzMyXTtyZXR1cm4gdFswXT4yNTV8fHRbMV0+MjU1fHx0WzJdPjI1NT8iIjoiG1tNIitsKHRbMF0pK2wodFsxXSkrbCh0WzJdKX0sU0dSOmZ1bmN0aW9uKGUpe3ZhciB0PTA9PT1lLmFjdGlvbiYmNCE9PWUuYnV0dG9uPyJtIjoiTSI7cmV0dXJuIhtbPCIrYyhlLCEwKSsiOyIrZS5jb2wrIjsiK2Uucm93K3R9fSxoPWZ1bmN0aW9uKCl7ZnVuY3Rpb24gZShlLHQpe3RoaXMuX2J1ZmZlclNlcnZpY2U9ZSx0aGlzLl9jb3JlU2VydmljZT10LHRoaXMuX3Byb3RvY29scz17fSx0aGlzLl9lbmNvZGluZ3M9e30sdGhpcy5fYWN0aXZlUHJvdG9jb2w9IiIsdGhpcy5fYWN0aXZlRW5jb2Rpbmc9IiIsdGhpcy5fb25Qcm90b2NvbENoYW5nZT1uZXcgcy5FdmVudEVtaXR0ZXIsdGhpcy5fbGFzdEV2ZW50PW51bGw7Zm9yKHZhciByPTAsaT1PYmplY3Qua2V5cyhhKTtyPGkubGVuZ3RoO3IrKyl7dmFyIG49aVtyXTt0aGlzLmFkZFByb3RvY29sKG4sYVtuXSl9Zm9yKHZhciBvPTAsYz1PYmplY3Qua2V5cyh1KTtvPGMubGVuZ3RoO28rKyl7dmFyIGw9Y1tvXTt0aGlzLmFkZEVuY29kaW5nKGwsdVtsXSl9dGhpcy5yZXNldCgpfXJldHVybiBlLnByb3RvdHlwZS5hZGRQcm90b2NvbD1mdW5jdGlvbihlLHQpe3RoaXMuX3Byb3RvY29sc1tlXT10fSxlLnByb3RvdHlwZS5hZGRFbmNvZGluZz1mdW5jdGlvbihlLHQpe3RoaXMuX2VuY29kaW5nc1tlXT10fSxPYmplY3QuZGVmaW5lUHJvcGVydHkoZS5wcm90b3R5cGUsImFjdGl2ZVByb3RvY29sIix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX2FjdGl2ZVByb3RvY29sfSxzZXQ6ZnVuY3Rpb24oZSl7aWYoIXRoaXMuX3Byb3RvY29sc1tlXSl0aHJvdyBuZXcgRXJyb3IoJ3Vua25vd24gcHJvdG9jb2wgIicrZSsnIicpO3RoaXMuX2FjdGl2ZVByb3RvY29sPWUsdGhpcy5fb25Qcm90b2NvbENoYW5nZS5maXJlKHRoaXMuX3Byb3RvY29sc1tlXS5ldmVudHMpfSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLE9iamVjdC5kZWZpbmVQcm9wZXJ0eShlLnByb3RvdHlwZSwiYXJlTW91c2VFdmVudHNBY3RpdmUiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gMCE9PXRoaXMuX3Byb3RvY29sc1t0aGlzLl9hY3RpdmVQcm90b2NvbF0uZXZlbnRzfSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLE9iamVjdC5kZWZpbmVQcm9wZXJ0eShlLnByb3RvdHlwZSwiYWN0aXZlRW5jb2RpbmciLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fYWN0aXZlRW5jb2Rpbmd9LHNldDpmdW5jdGlvbihlKXtpZighdGhpcy5fZW5jb2RpbmdzW2VdKXRocm93IG5ldyBFcnJvcigndW5rbm93biBlbmNvZGluZyAiJytlKyciJyk7dGhpcy5fYWN0aXZlRW5jb2Rpbmc9ZX0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxlLnByb3RvdHlwZS5yZXNldD1mdW5jdGlvbigpe3RoaXMuYWN0aXZlUHJvdG9jb2w9Ik5PTkUiLHRoaXMuYWN0aXZlRW5jb2Rpbmc9IkRFRkFVTFQiLHRoaXMuX2xhc3RFdmVudD1udWxsfSxPYmplY3QuZGVmaW5lUHJvcGVydHkoZS5wcm90b3R5cGUsIm9uUHJvdG9jb2xDaGFuZ2UiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fb25Qcm90b2NvbENoYW5nZS5ldmVudH0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxlLnByb3RvdHlwZS50cmlnZ2VyTW91c2VFdmVudD1mdW5jdGlvbihlKXtpZihlLmNvbDwwfHxlLmNvbD49dGhpcy5fYnVmZmVyU2VydmljZS5jb2xzfHxlLnJvdzwwfHxlLnJvdz49dGhpcy5fYnVmZmVyU2VydmljZS5yb3dzKXJldHVybiExO2lmKDQ9PT1lLmJ1dHRvbiYmMzI9PT1lLmFjdGlvbilyZXR1cm4hMTtpZigzPT09ZS5idXR0b24mJjMyIT09ZS5hY3Rpb24pcmV0dXJuITE7aWYoNCE9PWUuYnV0dG9uJiYoMj09PWUuYWN0aW9ufHwzPT09ZS5hY3Rpb24pKXJldHVybiExO2lmKGUuY29sKyssZS5yb3crKywzMj09PWUuYWN0aW9uJiZ0aGlzLl9sYXN0RXZlbnQmJnRoaXMuX2NvbXBhcmVFdmVudHModGhpcy5fbGFzdEV2ZW50LGUpKXJldHVybiExO2lmKCF0aGlzLl9wcm90b2NvbHNbdGhpcy5fYWN0aXZlUHJvdG9jb2xdLnJlc3RyaWN0KGUpKXJldHVybiExO3ZhciB0PXRoaXMuX2VuY29kaW5nc1t0aGlzLl9hY3RpdmVFbmNvZGluZ10oZSk7cmV0dXJuIHQmJigiREVGQVVMVCI9PT10aGlzLl9hY3RpdmVFbmNvZGluZz90aGlzLl9jb3JlU2VydmljZS50cmlnZ2VyQmluYXJ5RXZlbnQodCk6dGhpcy5fY29yZVNlcnZpY2UudHJpZ2dlckRhdGFFdmVudCh0LCEwKSksdGhpcy5fbGFzdEV2ZW50PWUsITB9LGUucHJvdG90eXBlLmV4cGxhaW5FdmVudHM9ZnVuY3Rpb24oZSl7cmV0dXJue2Rvd246ISEoMSZlKSx1cDohISgyJmUpLGRyYWc6ISEoNCZlKSxtb3ZlOiEhKDgmZSksd2hlZWw6ISEoMTYmZSl9fSxlLnByb3RvdHlwZS5fY29tcGFyZUV2ZW50cz1mdW5jdGlvbihlLHQpe3JldHVybiBlLmNvbD09PXQuY29sJiZlLnJvdz09PXQucm93JiZlLmJ1dHRvbj09PXQuYnV0dG9uJiZlLmFjdGlvbj09PXQuYWN0aW9uJiZlLmN0cmw9PT10LmN0cmwmJmUuYWx0PT09dC5hbHQmJmUuc2hpZnQ9PT10LnNoaWZ0fSxpKFtuKDAsby5JQnVmZmVyU2VydmljZSksbigxLG8uSUNvcmVTZXJ2aWNlKV0sZSl9KCk7dC5Db3JlTW91c2VTZXJ2aWNlPWh9LDY5NzU6ZnVuY3Rpb24oZSx0LHIpe3ZhciBpLG49dGhpcyYmdGhpcy5fX2V4dGVuZHN8fChpPWZ1bmN0aW9uKGUsdCl7cmV0dXJuIGk9T2JqZWN0LnNldFByb3RvdHlwZU9mfHx7X19wcm90b19fOltdfWluc3RhbmNlb2YgQXJyYXkmJmZ1bmN0aW9uKGUsdCl7ZS5fX3Byb3RvX189dH18fGZ1bmN0aW9uKGUsdCl7Zm9yKHZhciByIGluIHQpT2JqZWN0LnByb3RvdHlwZS5oYXNPd25Qcm9wZXJ0eS5jYWxsKHQscikmJihlW3JdPXRbcl0pfSxpKGUsdCl9LGZ1bmN0aW9uKGUsdCl7aWYoImZ1bmN0aW9uIiE9dHlwZW9mIHQmJm51bGwhPT10KXRocm93IG5ldyBUeXBlRXJyb3IoIkNsYXNzIGV4dGVuZHMgdmFsdWUgIitTdHJpbmcodCkrIiBpcyBub3QgYSBjb25zdHJ1Y3RvciBvciBudWxsIik7ZnVuY3Rpb24gcigpe3RoaXMuY29uc3RydWN0b3I9ZX1pKGUsdCksZS5wcm90b3R5cGU9bnVsbD09PXQ/T2JqZWN0LmNyZWF0ZSh0KTooci5wcm90b3R5cGU9dC5wcm90b3R5cGUsbmV3IHIpfSksbz10aGlzJiZ0aGlzLl9fZGVjb3JhdGV8fGZ1bmN0aW9uKGUsdCxyLGkpe3ZhciBuLG89YXJndW1lbnRzLmxlbmd0aCxzPW88Mz90Om51bGw9PT1pP2k9T2JqZWN0LmdldE93blByb3BlcnR5RGVzY3JpcHRvcih0LHIpOmk7aWYoIm9iamVjdCI9PXR5cGVvZiBSZWZsZWN0JiYiZnVuY3Rpb24iPT10eXBlb2YgUmVmbGVjdC5kZWNvcmF0ZSlzPVJlZmxlY3QuZGVjb3JhdGUoZSx0LHIsaSk7ZWxzZSBmb3IodmFyIGE9ZS5sZW5ndGgtMTthPj0wO2EtLSkobj1lW2FdKSYmKHM9KG88Mz9uKHMpOm8+Mz9uKHQscixzKTpuKHQscikpfHxzKTtyZXR1cm4gbz4zJiZzJiZPYmplY3QuZGVmaW5lUHJvcGVydHkodCxyLHMpLHN9LHM9dGhpcyYmdGhpcy5fX3BhcmFtfHxmdW5jdGlvbihlLHQpe3JldHVybiBmdW5jdGlvbihyLGkpe3QocixpLGUpfX07T2JqZWN0LmRlZmluZVByb3BlcnR5KHQsIl9fZXNNb2R1bGUiLHt2YWx1ZTohMH0pLHQuQ29yZVNlcnZpY2U9dm9pZCAwO3ZhciBhPXIoMjU4NSksYz1yKDg0NjApLGw9cigxNDM5KSx1PXIoODQ0KSxoPU9iamVjdC5mcmVlemUoe2luc2VydE1vZGU6ITF9KSxmPU9iamVjdC5mcmVlemUoe2FwcGxpY2F0aW9uQ3Vyc29yS2V5czohMSxhcHBsaWNhdGlvbktleXBhZDohMSxicmFja2V0ZWRQYXN0ZU1vZGU6ITEsb3JpZ2luOiExLHJldmVyc2VXcmFwYXJvdW5kOiExLHNlbmRGb2N1czohMSx3cmFwYXJvdW5kOiEwfSksXz1mdW5jdGlvbihlKXtmdW5jdGlvbiB0KHQscixpLG4pe3ZhciBvPWUuY2FsbCh0aGlzKXx8dGhpcztyZXR1cm4gby5fYnVmZmVyU2VydmljZT1yLG8uX2xvZ1NlcnZpY2U9aSxvLl9vcHRpb25zU2VydmljZT1uLG8uaXNDdXJzb3JJbml0aWFsaXplZD0hMSxvLmlzQ3Vyc29ySGlkZGVuPSExLG8uX29uRGF0YT1vLnJlZ2lzdGVyKG5ldyBjLkV2ZW50RW1pdHRlciksby5fb25Vc2VySW5wdXQ9by5yZWdpc3RlcihuZXcgYy5FdmVudEVtaXR0ZXIpLG8uX29uQmluYXJ5PW8ucmVnaXN0ZXIobmV3IGMuRXZlbnRFbWl0dGVyKSxvLl9zY3JvbGxUb0JvdHRvbT10LG8ucmVnaXN0ZXIoe2Rpc3Bvc2U6ZnVuY3Rpb24oKXtyZXR1cm4gby5fc2Nyb2xsVG9Cb3R0b209dm9pZCAwfX0pLG8ubW9kZXM9KDAsbC5jbG9uZSkoaCksby5kZWNQcml2YXRlTW9kZXM9KDAsbC5jbG9uZSkoZiksb31yZXR1cm4gbih0LGUpLE9iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LnByb3RvdHlwZSwib25EYXRhIix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX29uRGF0YS5ldmVudH0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxPYmplY3QuZGVmaW5lUHJvcGVydHkodC5wcm90b3R5cGUsIm9uVXNlcklucHV0Iix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX29uVXNlcklucHV0LmV2ZW50fSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLE9iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LnByb3RvdHlwZSwib25CaW5hcnkiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fb25CaW5hcnkuZXZlbnR9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksdC5wcm90b3R5cGUucmVzZXQ9ZnVuY3Rpb24oKXt0aGlzLm1vZGVzPSgwLGwuY2xvbmUpKGgpLHRoaXMuZGVjUHJpdmF0ZU1vZGVzPSgwLGwuY2xvbmUpKGYpfSx0LnByb3RvdHlwZS50cmlnZ2VyRGF0YUV2ZW50PWZ1bmN0aW9uKGUsdCl7aWYodm9pZCAwPT09dCYmKHQ9ITEpLCF0aGlzLl9vcHRpb25zU2VydmljZS5vcHRpb25zLmRpc2FibGVTdGRpbil7dmFyIHI9dGhpcy5fYnVmZmVyU2VydmljZS5idWZmZXI7ci55YmFzZSE9PXIueWRpc3AmJnRoaXMuX3Njcm9sbFRvQm90dG9tKCksdCYmdGhpcy5fb25Vc2VySW5wdXQuZmlyZSgpLHRoaXMuX2xvZ1NlcnZpY2UuZGVidWcoJ3NlbmRpbmcgZGF0YSAiJytlKyciJywoZnVuY3Rpb24oKXtyZXR1cm4gZS5zcGxpdCgiIikubWFwKChmdW5jdGlvbihlKXtyZXR1cm4gZS5jaGFyQ29kZUF0KDApfSkpfSkpLHRoaXMuX29uRGF0YS5maXJlKGUpfX0sdC5wcm90b3R5cGUudHJpZ2dlckJpbmFyeUV2ZW50PWZ1bmN0aW9uKGUpe3RoaXMuX29wdGlvbnNTZXJ2aWNlLm9wdGlvbnMuZGlzYWJsZVN0ZGlufHwodGhpcy5fbG9nU2VydmljZS5kZWJ1Zygnc2VuZGluZyBiaW5hcnkgIicrZSsnIicsKGZ1bmN0aW9uKCl7cmV0dXJuIGUuc3BsaXQoIiIpLm1hcCgoZnVuY3Rpb24oZSl7cmV0dXJuIGUuY2hhckNvZGVBdCgwKX0pKX0pKSx0aGlzLl9vbkJpbmFyeS5maXJlKGUpKX0sbyhbcygxLGEuSUJ1ZmZlclNlcnZpY2UpLHMoMixhLklMb2dTZXJ2aWNlKSxzKDMsYS5JT3B0aW9uc1NlcnZpY2UpXSx0KX0odS5EaXNwb3NhYmxlKTt0LkNvcmVTZXJ2aWNlPV99LDM3MzA6ZnVuY3Rpb24oZSx0LHIpe3ZhciBpPXRoaXMmJnRoaXMuX19kZWNvcmF0ZXx8ZnVuY3Rpb24oZSx0LHIsaSl7dmFyIG4sbz1hcmd1bWVudHMubGVuZ3RoLHM9bzwzP3Q6bnVsbD09PWk/aT1PYmplY3QuZ2V0T3duUHJvcGVydHlEZXNjcmlwdG9yKHQscik6aTtpZigib2JqZWN0Ij09dHlwZW9mIFJlZmxlY3QmJiJmdW5jdGlvbiI9PXR5cGVvZiBSZWZsZWN0LmRlY29yYXRlKXM9UmVmbGVjdC5kZWNvcmF0ZShlLHQscixpKTtlbHNlIGZvcih2YXIgYT1lLmxlbmd0aC0xO2E+PTA7YS0tKShuPWVbYV0pJiYocz0obzwzP24ocyk6bz4zP24odCxyLHMpOm4odCxyKSl8fHMpO3JldHVybiBvPjMmJnMmJk9iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LHIscyksc30sbj10aGlzJiZ0aGlzLl9fcGFyYW18fGZ1bmN0aW9uKGUsdCl7cmV0dXJuIGZ1bmN0aW9uKHIsaSl7dChyLGksZSl9fTtPYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC5EaXJ0eVJvd1NlcnZpY2U9dm9pZCAwO3ZhciBvPXIoMjU4NSkscz1mdW5jdGlvbigpe2Z1bmN0aW9uIGUoZSl7dGhpcy5fYnVmZmVyU2VydmljZT1lLHRoaXMuY2xlYXJSYW5nZSgpfXJldHVybiBPYmplY3QuZGVmaW5lUHJvcGVydHkoZS5wcm90b3R5cGUsInN0YXJ0Iix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX3N0YXJ0fSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLE9iamVjdC5kZWZpbmVQcm9wZXJ0eShlLnByb3RvdHlwZSwiZW5kIix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX2VuZH0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxlLnByb3RvdHlwZS5jbGVhclJhbmdlPWZ1bmN0aW9uKCl7dGhpcy5fc3RhcnQ9dGhpcy5fYnVmZmVyU2VydmljZS5idWZmZXIueSx0aGlzLl9lbmQ9dGhpcy5fYnVmZmVyU2VydmljZS5idWZmZXIueX0sZS5wcm90b3R5cGUubWFya0RpcnR5PWZ1bmN0aW9uKGUpe2U8dGhpcy5fc3RhcnQ/dGhpcy5fc3RhcnQ9ZTplPnRoaXMuX2VuZCYmKHRoaXMuX2VuZD1lKX0sZS5wcm90b3R5cGUubWFya1JhbmdlRGlydHk9ZnVuY3Rpb24oZSx0KXtpZihlPnQpe3ZhciByPWU7ZT10LHQ9cn1lPHRoaXMuX3N0YXJ0JiYodGhpcy5fc3RhcnQ9ZSksdD50aGlzLl9lbmQmJih0aGlzLl9lbmQ9dCl9LGUucHJvdG90eXBlLm1hcmtBbGxEaXJ0eT1mdW5jdGlvbigpe3RoaXMubWFya1JhbmdlRGlydHkoMCx0aGlzLl9idWZmZXJTZXJ2aWNlLnJvd3MtMSl9LGkoW24oMCxvLklCdWZmZXJTZXJ2aWNlKV0sZSl9KCk7dC5EaXJ0eVJvd1NlcnZpY2U9c30sNDM0ODpmdW5jdGlvbihlLHQscil7dmFyIGk9dGhpcyYmdGhpcy5fX3NwcmVhZEFycmF5fHxmdW5jdGlvbihlLHQscil7aWYocnx8Mj09PWFyZ3VtZW50cy5sZW5ndGgpZm9yKHZhciBpLG49MCxvPXQubGVuZ3RoO248bztuKyspIWkmJm4gaW4gdHx8KGl8fChpPUFycmF5LnByb3RvdHlwZS5zbGljZS5jYWxsKHQsMCxuKSksaVtuXT10W25dKTtyZXR1cm4gZS5jb25jYXQoaXx8QXJyYXkucHJvdG90eXBlLnNsaWNlLmNhbGwodCkpfTtPYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC5JbnN0YW50aWF0aW9uU2VydmljZT10LlNlcnZpY2VDb2xsZWN0aW9uPXZvaWQgMDt2YXIgbj1yKDI1ODUpLG89cig4MzQzKSxzPWZ1bmN0aW9uKCl7ZnVuY3Rpb24gZSgpe2Zvcih2YXIgZT1bXSx0PTA7dDxhcmd1bWVudHMubGVuZ3RoO3QrKyllW3RdPWFyZ3VtZW50c1t0XTt0aGlzLl9lbnRyaWVzPW5ldyBNYXA7Zm9yKHZhciByPTAsaT1lO3I8aS5sZW5ndGg7cisrKXt2YXIgbj1pW3JdLG89blswXSxzPW5bMV07dGhpcy5zZXQobyxzKX19cmV0dXJuIGUucHJvdG90eXBlLnNldD1mdW5jdGlvbihlLHQpe3ZhciByPXRoaXMuX2VudHJpZXMuZ2V0KGUpO3JldHVybiB0aGlzLl9lbnRyaWVzLnNldChlLHQpLHJ9LGUucHJvdG90eXBlLmZvckVhY2g9ZnVuY3Rpb24oZSl7dGhpcy5fZW50cmllcy5mb3JFYWNoKChmdW5jdGlvbih0LHIpe3JldHVybiBlKHIsdCl9KSl9LGUucHJvdG90eXBlLmhhcz1mdW5jdGlvbihlKXtyZXR1cm4gdGhpcy5fZW50cmllcy5oYXMoZSl9LGUucHJvdG90eXBlLmdldD1mdW5jdGlvbihlKXtyZXR1cm4gdGhpcy5fZW50cmllcy5nZXQoZSl9LGV9KCk7dC5TZXJ2aWNlQ29sbGVjdGlvbj1zO3ZhciBhPWZ1bmN0aW9uKCl7ZnVuY3Rpb24gZSgpe3RoaXMuX3NlcnZpY2VzPW5ldyBzLHRoaXMuX3NlcnZpY2VzLnNldChuLklJbnN0YW50aWF0aW9uU2VydmljZSx0aGlzKX1yZXR1cm4gZS5wcm90b3R5cGUuc2V0U2VydmljZT1mdW5jdGlvbihlLHQpe3RoaXMuX3NlcnZpY2VzLnNldChlLHQpfSxlLnByb3RvdHlwZS5nZXRTZXJ2aWNlPWZ1bmN0aW9uKGUpe3JldHVybiB0aGlzLl9zZXJ2aWNlcy5nZXQoZSl9LGUucHJvdG90eXBlLmNyZWF0ZUluc3RhbmNlPWZ1bmN0aW9uKGUpe2Zvcih2YXIgdD1bXSxyPTE7cjxhcmd1bWVudHMubGVuZ3RoO3IrKyl0W3ItMV09YXJndW1lbnRzW3JdO2Zvcih2YXIgbj0oMCxvLmdldFNlcnZpY2VEZXBlbmRlbmNpZXMpKGUpLnNvcnQoKGZ1bmN0aW9uKGUsdCl7cmV0dXJuIGUuaW5kZXgtdC5pbmRleH0pKSxzPVtdLGE9MCxjPW47YTxjLmxlbmd0aDthKyspe3ZhciBsPWNbYV0sdT10aGlzLl9zZXJ2aWNlcy5nZXQobC5pZCk7aWYoIXUpdGhyb3cgbmV3IEVycm9yKCJbY3JlYXRlSW5zdGFuY2VdICIrZS5uYW1lKyIgZGVwZW5kcyBvbiBVTktOT1dOIHNlcnZpY2UgIitsLmlkKyIuIik7cy5wdXNoKHUpfXZhciBoPW4ubGVuZ3RoPjA/blswXS5pbmRleDp0Lmxlbmd0aDtpZih0Lmxlbmd0aCE9PWgpdGhyb3cgbmV3IEVycm9yKCJbY3JlYXRlSW5zdGFuY2VdIEZpcnN0IHNlcnZpY2UgZGVwZW5kZW5jeSBvZiAiK2UubmFtZSsiIGF0IHBvc2l0aW9uICIrKGgrMSkrIiBjb25mbGljdHMgd2l0aCAiK3QubGVuZ3RoKyIgc3RhdGljIGFyZ3VtZW50cyIpO3JldHVybiBuZXcoZS5iaW5kLmFwcGx5KGUsaShbdm9pZCAwXSxpKGkoW10sdCwhMCkscywhMCksITEpKSl9LGV9KCk7dC5JbnN0YW50aWF0aW9uU2VydmljZT1hfSw3ODY2OmZ1bmN0aW9uKGUsdCxyKXt2YXIgaT10aGlzJiZ0aGlzLl9fZGVjb3JhdGV8fGZ1bmN0aW9uKGUsdCxyLGkpe3ZhciBuLG89YXJndW1lbnRzLmxlbmd0aCxzPW88Mz90Om51bGw9PT1pP2k9T2JqZWN0LmdldE93blByb3BlcnR5RGVzY3JpcHRvcih0LHIpOmk7aWYoIm9iamVjdCI9PXR5cGVvZiBSZWZsZWN0JiYiZnVuY3Rpb24iPT10eXBlb2YgUmVmbGVjdC5kZWNvcmF0ZSlzPVJlZmxlY3QuZGVjb3JhdGUoZSx0LHIsaSk7ZWxzZSBmb3IodmFyIGE9ZS5sZW5ndGgtMTthPj0wO2EtLSkobj1lW2FdKSYmKHM9KG88Mz9uKHMpOm8+Mz9uKHQscixzKTpuKHQscikpfHxzKTtyZXR1cm4gbz4zJiZzJiZPYmplY3QuZGVmaW5lUHJvcGVydHkodCxyLHMpLHN9LG49dGhpcyYmdGhpcy5fX3BhcmFtfHxmdW5jdGlvbihlLHQpe3JldHVybiBmdW5jdGlvbihyLGkpe3QocixpLGUpfX0sbz10aGlzJiZ0aGlzLl9fc3ByZWFkQXJyYXl8fGZ1bmN0aW9uKGUsdCxyKXtpZihyfHwyPT09YXJndW1lbnRzLmxlbmd0aClmb3IodmFyIGksbj0wLG89dC5sZW5ndGg7bjxvO24rKykhaSYmbiBpbiB0fHwoaXx8KGk9QXJyYXkucHJvdG90eXBlLnNsaWNlLmNhbGwodCwwLG4pKSxpW25dPXRbbl0pO3JldHVybiBlLmNvbmNhdChpfHxBcnJheS5wcm90b3R5cGUuc2xpY2UuY2FsbCh0KSl9O09iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LCJfX2VzTW9kdWxlIix7dmFsdWU6ITB9KSx0LkxvZ1NlcnZpY2U9dm9pZCAwO3ZhciBzPXIoMjU4NSksYT17ZGVidWc6cy5Mb2dMZXZlbEVudW0uREVCVUcsaW5mbzpzLkxvZ0xldmVsRW51bS5JTkZPLHdhcm46cy5Mb2dMZXZlbEVudW0uV0FSTixlcnJvcjpzLkxvZ0xldmVsRW51bS5FUlJPUixvZmY6cy5Mb2dMZXZlbEVudW0uT0ZGfSxjPWZ1bmN0aW9uKCl7ZnVuY3Rpb24gZShlKXt2YXIgdD10aGlzO3RoaXMuX29wdGlvbnNTZXJ2aWNlPWUsdGhpcy5sb2dMZXZlbD1zLkxvZ0xldmVsRW51bS5PRkYsdGhpcy5fdXBkYXRlTG9nTGV2ZWwoKSx0aGlzLl9vcHRpb25zU2VydmljZS5vbk9wdGlvbkNoYW5nZSgoZnVuY3Rpb24oZSl7ImxvZ0xldmVsIj09PWUmJnQuX3VwZGF0ZUxvZ0xldmVsKCl9KSl9cmV0dXJuIGUucHJvdG90eXBlLl91cGRhdGVMb2dMZXZlbD1mdW5jdGlvbigpe3RoaXMubG9nTGV2ZWw9YVt0aGlzLl9vcHRpb25zU2VydmljZS5vcHRpb25zLmxvZ0xldmVsXX0sZS5wcm90b3R5cGUuX2V2YWxMYXp5T3B0aW9uYWxQYXJhbXM9ZnVuY3Rpb24oZSl7Zm9yKHZhciB0PTA7dDxlLmxlbmd0aDt0KyspImZ1bmN0aW9uIj09dHlwZW9mIGVbdF0mJihlW3RdPWVbdF0oKSl9LGUucHJvdG90eXBlLl9sb2c9ZnVuY3Rpb24oZSx0LHIpe3RoaXMuX2V2YWxMYXp5T3B0aW9uYWxQYXJhbXMociksZS5jYWxsLmFwcGx5KGUsbyhbY29uc29sZSwieHRlcm0uanM6ICIrdF0sciwhMSkpfSxlLnByb3RvdHlwZS5kZWJ1Zz1mdW5jdGlvbihlKXtmb3IodmFyIHQ9W10scj0xO3I8YXJndW1lbnRzLmxlbmd0aDtyKyspdFtyLTFdPWFyZ3VtZW50c1tyXTt0aGlzLmxvZ0xldmVsPD1zLkxvZ0xldmVsRW51bS5ERUJVRyYmdGhpcy5fbG9nKGNvbnNvbGUubG9nLGUsdCl9LGUucHJvdG90eXBlLmluZm89ZnVuY3Rpb24oZSl7Zm9yKHZhciB0PVtdLHI9MTtyPGFyZ3VtZW50cy5sZW5ndGg7cisrKXRbci0xXT1hcmd1bWVudHNbcl07dGhpcy5sb2dMZXZlbDw9cy5Mb2dMZXZlbEVudW0uSU5GTyYmdGhpcy5fbG9nKGNvbnNvbGUuaW5mbyxlLHQpfSxlLnByb3RvdHlwZS53YXJuPWZ1bmN0aW9uKGUpe2Zvcih2YXIgdD1bXSxyPTE7cjxhcmd1bWVudHMubGVuZ3RoO3IrKyl0W3ItMV09YXJndW1lbnRzW3JdO3RoaXMubG9nTGV2ZWw8PXMuTG9nTGV2ZWxFbnVtLldBUk4mJnRoaXMuX2xvZyhjb25zb2xlLndhcm4sZSx0KX0sZS5wcm90b3R5cGUuZXJyb3I9ZnVuY3Rpb24oZSl7Zm9yKHZhciB0PVtdLHI9MTtyPGFyZ3VtZW50cy5sZW5ndGg7cisrKXRbci0xXT1hcmd1bWVudHNbcl07dGhpcy5sb2dMZXZlbDw9cy5Mb2dMZXZlbEVudW0uRVJST1ImJnRoaXMuX2xvZyhjb25zb2xlLmVycm9yLGUsdCl9LGkoW24oMCxzLklPcHRpb25zU2VydmljZSldLGUpfSgpO3QuTG9nU2VydmljZT1jfSw3MzAyOmZ1bmN0aW9uKGUsdCxyKXt2YXIgaT10aGlzJiZ0aGlzLl9fYXNzaWdufHxmdW5jdGlvbigpe3JldHVybiBpPU9iamVjdC5hc3NpZ258fGZ1bmN0aW9uKGUpe2Zvcih2YXIgdCxyPTEsaT1hcmd1bWVudHMubGVuZ3RoO3I8aTtyKyspZm9yKHZhciBuIGluIHQ9YXJndW1lbnRzW3JdKU9iamVjdC5wcm90b3R5cGUuaGFzT3duUHJvcGVydHkuY2FsbCh0LG4pJiYoZVtuXT10W25dKTtyZXR1cm4gZX0saS5hcHBseSh0aGlzLGFyZ3VtZW50cyl9O09iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LCJfX2VzTW9kdWxlIix7dmFsdWU6ITB9KSx0Lk9wdGlvbnNTZXJ2aWNlPXQuREVGQVVMVF9PUFRJT05TPXQuREVGQVVMVF9CRUxMX1NPVU5EPXZvaWQgMDt2YXIgbj1yKDg0NjApLG89cig2MTE0KTt0LkRFRkFVTFRfQkVMTF9TT1VORD0iZGF0YTphdWRpby9tcDM7YmFzZTY0LFNVUXpCQUFBQUFBQUkxUlRVMFVBQUFBUEFBQURUR0YyWmpVNExqTXlMakV3TkFBQUFBQUFBQUFBQUFBQS8vdFF4QUFEQjhBaFNteGhJSUVWQ1NpSnJEQ1FCVGN1M1VyQUl3VWRrUmdRYkZBWkMxQ1FFd1RKOW1qUnZCQTRVT0xEOG5LVk9XZmgrVWxLM3ovMTc3T1hyZk9kS2w3cHluM1hmLy9XcmV5VFJVb0FXZ0Jna09BR2JaSEJnRzFPRjZ6TTgyRFdiWmFVbU1CcHRnUWhHanN5WXFjOWFlOVhGejI4MDk0OE5NQldJbmxqeXpzTlJGTFBXZG5aR1dyZGREc2pLMXVudVNyVk45akpzSzhLdVF0UUN0TUJqQ0V0SW1JU2ROS0pPb3BJcEJGcE5TTWJJSENTUnBSUjVpYWtqVGl5ekxoY2hVVUJ3Q2d5S2l3ZUJ2LzdVc1FiZzhpc1ZOb01QTWpBQUFBMGdBQUFCRVZGR21ncUsvLy8vOWJQLzZYQ3lreEJUVVV6TGpFd01LcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXEiLHQuREVGQVVMVF9PUFRJT05TPXtjb2xzOjgwLHJvd3M6MjQsY3Vyc29yQmxpbms6ITEsY3Vyc29yU3R5bGU6ImJsb2NrIixjdXJzb3JXaWR0aDoxLGN1c3RvbUdseXBoczohMCxiZWxsU291bmQ6dC5ERUZBVUxUX0JFTExfU09VTkQsYmVsbFN0eWxlOiJub25lIixkcmF3Qm9sZFRleHRJbkJyaWdodENvbG9yczohMCxmYXN0U2Nyb2xsTW9kaWZpZXI6ImFsdCIsZmFzdFNjcm9sbFNlbnNpdGl2aXR5OjUsZm9udEZhbWlseToiY291cmllci1uZXcsIGNvdXJpZXIsIG1vbm9zcGFjZSIsZm9udFNpemU6MTUsZm9udFdlaWdodDoibm9ybWFsIixmb250V2VpZ2h0Qm9sZDoiYm9sZCIsbGluZUhlaWdodDoxLGxpbmtUb29sdGlwSG92ZXJEdXJhdGlvbjo1MDAsbGV0dGVyU3BhY2luZzowLGxvZ0xldmVsOiJpbmZvIixzY3JvbGxiYWNrOjFlMyxzY3JvbGxTZW5zaXRpdml0eToxLHNjcmVlblJlYWRlck1vZGU6ITEsbWFjT3B0aW9uSXNNZXRhOiExLG1hY09wdGlvbkNsaWNrRm9yY2VzU2VsZWN0aW9uOiExLG1pbmltdW1Db250cmFzdFJhdGlvOjEsZGlzYWJsZVN0ZGluOiExLGFsbG93UHJvcG9zZWRBcGk6ITAsYWxsb3dUcmFuc3BhcmVuY3k6ITEsdGFiU3RvcFdpZHRoOjgsdGhlbWU6e30scmlnaHRDbGlja1NlbGVjdHNXb3JkOm8uaXNNYWMscmVuZGVyZXJUeXBlOiJjYW52YXMiLHdpbmRvd09wdGlvbnM6e30sd2luZG93c01vZGU6ITEsd29yZFNlcGFyYXRvcjoiICgpW117fScsXCJgIixhbHRDbGlja01vdmVzQ3Vyc29yOiEwLGNvbnZlcnRFb2w6ITEsdGVybU5hbWU6Inh0ZXJtIixjYW5jZWxFdmVudHM6ITF9O3ZhciBzPVsibm9ybWFsIiwiYm9sZCIsIjEwMCIsIjIwMCIsIjMwMCIsIjQwMCIsIjUwMCIsIjYwMCIsIjcwMCIsIjgwMCIsIjkwMCJdLGE9ZnVuY3Rpb24oKXtmdW5jdGlvbiBlKGUpe2Zvcih2YXIgciBpbiB0aGlzLl9vbk9wdGlvbkNoYW5nZT1uZXcgbi5FdmVudEVtaXR0ZXIsdGhpcy5fb3B0aW9ucz1pKHt9LHQuREVGQVVMVF9PUFRJT05TKSxlKWlmKHIgaW4gdGhpcy5fb3B0aW9ucyl0cnl7dmFyIG89ZVtyXTt0aGlzLl9vcHRpb25zW3JdPXRoaXMuX3Nhbml0aXplQW5kVmFsaWRhdGVPcHRpb24ocixvKX1jYXRjaChlKXtjb25zb2xlLmVycm9yKGUpfXRoaXMub3B0aW9ucz10aGlzLl9zZXR1cE9wdGlvbnModGhpcy5fb3B0aW9ucyl9cmV0dXJuIE9iamVjdC5kZWZpbmVQcm9wZXJ0eShlLnByb3RvdHlwZSwib25PcHRpb25DaGFuZ2UiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fb25PcHRpb25DaGFuZ2UuZXZlbnR9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksZS5wcm90b3R5cGUuX3NldHVwT3B0aW9ucz1mdW5jdGlvbihlKXt2YXIgcj10aGlzLG49aSh7fSxlKSxvPWZ1bmN0aW9uKGUpe09iamVjdC5kZWZpbmVQcm9wZXJ0eShuLGUse2dldDpmdW5jdGlvbigpe2lmKCEoZSBpbiB0LkRFRkFVTFRfT1BUSU9OUykpdGhyb3cgbmV3IEVycm9yKCdObyBvcHRpb24gd2l0aCBrZXkgIicrZSsnIicpO3JldHVybiByLl9vcHRpb25zW2VdfSxzZXQ6ZnVuY3Rpb24oaSl7aWYoIShlIGluIHQuREVGQVVMVF9PUFRJT05TKSl0aHJvdyBuZXcgRXJyb3IoJ05vIG9wdGlvbiB3aXRoIGtleSAiJytlKyciJyk7aT1yLl9zYW5pdGl6ZUFuZFZhbGlkYXRlT3B0aW9uKGUsaSksci5fb3B0aW9uc1tlXSE9PWkmJihyLl9vcHRpb25zW2VdPWksci5fb25PcHRpb25DaGFuZ2UuZmlyZShlKSl9fSl9O2Zvcih2YXIgcyBpbiBuKW8ocyk7cmV0dXJuIG59LGUucHJvdG90eXBlLnNldE9wdGlvbj1mdW5jdGlvbihlLHQpe3RoaXMub3B0aW9uc1tlXT10fSxlLnByb3RvdHlwZS5fc2FuaXRpemVBbmRWYWxpZGF0ZU9wdGlvbj1mdW5jdGlvbihlLHIpe3N3aXRjaChlKXtjYXNlImJlbGxTdHlsZSI6Y2FzZSJjdXJzb3JTdHlsZSI6Y2FzZSJyZW5kZXJlclR5cGUiOmNhc2Uid29yZFNlcGFyYXRvciI6cnx8KHI9dC5ERUZBVUxUX09QVElPTlNbZV0pO2JyZWFrO2Nhc2UiZm9udFdlaWdodCI6Y2FzZSJmb250V2VpZ2h0Qm9sZCI6aWYoIm51bWJlciI9PXR5cGVvZiByJiYxPD1yJiZyPD0xZTMpYnJlYWs7cj1zLmluY2x1ZGVzKHIpP3I6dC5ERUZBVUxUX09QVElPTlNbZV07YnJlYWs7Y2FzZSJjdXJzb3JXaWR0aCI6cj1NYXRoLmZsb29yKHIpO2Nhc2UibGluZUhlaWdodCI6Y2FzZSJ0YWJTdG9wV2lkdGgiOmlmKHI8MSl0aHJvdyBuZXcgRXJyb3IoZSsiIGNhbm5vdCBiZSBsZXNzIHRoYW4gMSwgdmFsdWU6ICIrcik7YnJlYWs7Y2FzZSJtaW5pbXVtQ29udHJhc3RSYXRpbyI6cj1NYXRoLm1heCgxLE1hdGgubWluKDIxLE1hdGgucm91bmQoMTAqcikvMTApKTticmVhaztjYXNlInNjcm9sbGJhY2siOmlmKChyPU1hdGgubWluKHIsNDI5NDk2NzI5NSkpPDApdGhyb3cgbmV3IEVycm9yKGUrIiBjYW5ub3QgYmUgbGVzcyB0aGFuIDAsIHZhbHVlOiAiK3IpO2JyZWFrO2Nhc2UiZmFzdFNjcm9sbFNlbnNpdGl2aXR5IjpjYXNlInNjcm9sbFNlbnNpdGl2aXR5IjppZihyPD0wKXRocm93IG5ldyBFcnJvcihlKyIgY2Fubm90IGJlIGxlc3MgdGhhbiBvciBlcXVhbCB0byAwLCB2YWx1ZTogIityKTtjYXNlInJvd3MiOmNhc2UiY29scyI6aWYoIXImJjAhPT1yKXRocm93IG5ldyBFcnJvcihlKyIgbXVzdCBiZSBudW1lcmljLCB2YWx1ZTogIityKX1yZXR1cm4gcn0sZS5wcm90b3R5cGUuZ2V0T3B0aW9uPWZ1bmN0aW9uKGUpe3JldHVybiB0aGlzLm9wdGlvbnNbZV19LGV9KCk7dC5PcHRpb25zU2VydmljZT1hfSw4MzQzOihlLHQpPT57ZnVuY3Rpb24gcihlLHQscil7dC5kaSR0YXJnZXQ9PT10P3QuZGkkZGVwZW5kZW5jaWVzLnB1c2goe2lkOmUsaW5kZXg6cn0pOih0LmRpJGRlcGVuZGVuY2llcz1be2lkOmUsaW5kZXg6cn1dLHQuZGkkdGFyZ2V0PXQpfU9iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LCJfX2VzTW9kdWxlIix7dmFsdWU6ITB9KSx0LmNyZWF0ZURlY29yYXRvcj10LmdldFNlcnZpY2VEZXBlbmRlbmNpZXM9dC5zZXJ2aWNlUmVnaXN0cnk9dm9pZCAwLHQuc2VydmljZVJlZ2lzdHJ5PW5ldyBNYXAsdC5nZXRTZXJ2aWNlRGVwZW5kZW5jaWVzPWZ1bmN0aW9uKGUpe3JldHVybiBlLmRpJGRlcGVuZGVuY2llc3x8W119LHQuY3JlYXRlRGVjb3JhdG9yPWZ1bmN0aW9uKGUpe2lmKHQuc2VydmljZVJlZ2lzdHJ5LmhhcyhlKSlyZXR1cm4gdC5zZXJ2aWNlUmVnaXN0cnkuZ2V0KGUpO3ZhciBpPWZ1bmN0aW9uKGUsdCxuKXtpZigzIT09YXJndW1lbnRzLmxlbmd0aCl0aHJvdyBuZXcgRXJyb3IoIkBJU2VydmljZU5hbWUtZGVjb3JhdG9yIGNhbiBvbmx5IGJlIHVzZWQgdG8gZGVjb3JhdGUgYSBwYXJhbWV0ZXIiKTtyKGksZSxuKX07cmV0dXJuIGkudG9TdHJpbmc9ZnVuY3Rpb24oKXtyZXR1cm4gZX0sdC5zZXJ2aWNlUmVnaXN0cnkuc2V0KGUsaSksaX19LDI1ODU6KGUsdCxyKT0+e09iamVjdC5kZWZpbmVQcm9wZXJ0eSh0LCJfX2VzTW9kdWxlIix7dmFsdWU6ITB9KSx0LklVbmljb2RlU2VydmljZT10LklPcHRpb25zU2VydmljZT10LklMb2dTZXJ2aWNlPXQuTG9nTGV2ZWxFbnVtPXQuSUluc3RhbnRpYXRpb25TZXJ2aWNlPXQuSURpcnR5Um93U2VydmljZT10LklDaGFyc2V0U2VydmljZT10LklDb3JlU2VydmljZT10LklDb3JlTW91c2VTZXJ2aWNlPXQuSUJ1ZmZlclNlcnZpY2U9dm9pZCAwO3ZhciBpLG49cig4MzQzKTt0LklCdWZmZXJTZXJ2aWNlPSgwLG4uY3JlYXRlRGVjb3JhdG9yKSgiQnVmZmVyU2VydmljZSIpLHQuSUNvcmVNb3VzZVNlcnZpY2U9KDAsbi5jcmVhdGVEZWNvcmF0b3IpKCJDb3JlTW91c2VTZXJ2aWNlIiksdC5JQ29yZVNlcnZpY2U9KDAsbi5jcmVhdGVEZWNvcmF0b3IpKCJDb3JlU2VydmljZSIpLHQuSUNoYXJzZXRTZXJ2aWNlPSgwLG4uY3JlYXRlRGVjb3JhdG9yKSgiQ2hhcnNldFNlcnZpY2UiKSx0LklEaXJ0eVJvd1NlcnZpY2U9KDAsbi5jcmVhdGVEZWNvcmF0b3IpKCJEaXJ0eVJvd1NlcnZpY2UiKSx0LklJbnN0YW50aWF0aW9uU2VydmljZT0oMCxuLmNyZWF0ZURlY29yYXRvcikoIkluc3RhbnRpYXRpb25TZXJ2aWNlIiksKGk9dC5Mb2dMZXZlbEVudW18fCh0LkxvZ0xldmVsRW51bT17fSkpW2kuREVCVUc9MF09IkRFQlVHIixpW2kuSU5GTz0xXT0iSU5GTyIsaVtpLldBUk49Ml09IldBUk4iLGlbaS5FUlJPUj0zXT0iRVJST1IiLGlbaS5PRkY9NF09Ik9GRiIsdC5JTG9nU2VydmljZT0oMCxuLmNyZWF0ZURlY29yYXRvcikoIkxvZ1NlcnZpY2UiKSx0LklPcHRpb25zU2VydmljZT0oMCxuLmNyZWF0ZURlY29yYXRvcikoIk9wdGlvbnNTZXJ2aWNlIiksdC5JVW5pY29kZVNlcnZpY2U9KDAsbi5jcmVhdGVEZWNvcmF0b3IpKCJVbmljb2RlU2VydmljZSIpfSwxNDgwOihlLHQscik9PntPYmplY3QuZGVmaW5lUHJvcGVydHkodCwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksdC5Vbmljb2RlU2VydmljZT12b2lkIDA7dmFyIGk9cig4NDYwKSxuPXIoMjI1KSxvPWZ1bmN0aW9uKCl7ZnVuY3Rpb24gZSgpe3RoaXMuX3Byb3ZpZGVycz1PYmplY3QuY3JlYXRlKG51bGwpLHRoaXMuX2FjdGl2ZT0iIix0aGlzLl9vbkNoYW5nZT1uZXcgaS5FdmVudEVtaXR0ZXI7dmFyIGU9bmV3IG4uVW5pY29kZVY2O3RoaXMucmVnaXN0ZXIoZSksdGhpcy5fYWN0aXZlPWUudmVyc2lvbix0aGlzLl9hY3RpdmVQcm92aWRlcj1lfXJldHVybiBPYmplY3QuZGVmaW5lUHJvcGVydHkoZS5wcm90b3R5cGUsIm9uQ2hhbmdlIix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX29uQ2hhbmdlLmV2ZW50fSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLE9iamVjdC5kZWZpbmVQcm9wZXJ0eShlLnByb3RvdHlwZSwidmVyc2lvbnMiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gT2JqZWN0LmtleXModGhpcy5fcHJvdmlkZXJzKX0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxPYmplY3QuZGVmaW5lUHJvcGVydHkoZS5wcm90b3R5cGUsImFjdGl2ZVZlcnNpb24iLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fYWN0aXZlfSxzZXQ6ZnVuY3Rpb24oZSl7aWYoIXRoaXMuX3Byb3ZpZGVyc1tlXSl0aHJvdyBuZXcgRXJyb3IoJ3Vua25vd24gVW5pY29kZSB2ZXJzaW9uICInK2UrJyInKTt0aGlzLl9hY3RpdmU9ZSx0aGlzLl9hY3RpdmVQcm92aWRlcj10aGlzLl9wcm92aWRlcnNbZV0sdGhpcy5fb25DaGFuZ2UuZmlyZShlKX0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxlLnByb3RvdHlwZS5yZWdpc3Rlcj1mdW5jdGlvbihlKXt0aGlzLl9wcm92aWRlcnNbZS52ZXJzaW9uXT1lfSxlLnByb3RvdHlwZS53Y3dpZHRoPWZ1bmN0aW9uKGUpe3JldHVybiB0aGlzLl9hY3RpdmVQcm92aWRlci53Y3dpZHRoKGUpfSxlLnByb3RvdHlwZS5nZXRTdHJpbmdDZWxsV2lkdGg9ZnVuY3Rpb24oZSl7Zm9yKHZhciB0PTAscj1lLmxlbmd0aCxpPTA7aTxyOysraSl7dmFyIG49ZS5jaGFyQ29kZUF0KGkpO2lmKDU1Mjk2PD1uJiZuPD01NjMxOSl7aWYoKytpPj1yKXJldHVybiB0K3RoaXMud2N3aWR0aChuKTt2YXIgbz1lLmNoYXJDb2RlQXQoaSk7NTYzMjA8PW8mJm88PTU3MzQzP249MTAyNCoobi01NTI5Nikrby01NjMyMCs2NTUzNjp0Kz10aGlzLndjd2lkdGgobyl9dCs9dGhpcy53Y3dpZHRoKG4pfXJldHVybiB0fSxlfSgpO3QuVW5pY29kZVNlcnZpY2U9b319LHQ9e307ZnVuY3Rpb24gcihpKXt2YXIgbj10W2ldO2lmKHZvaWQgMCE9PW4pcmV0dXJuIG4uZXhwb3J0czt2YXIgbz10W2ldPXtleHBvcnRzOnt9fTtyZXR1cm4gZVtpXS5jYWxsKG8uZXhwb3J0cyxvLG8uZXhwb3J0cyxyKSxvLmV4cG9ydHN9dmFyIGk9e307cmV0dXJuKCgpPT57dmFyIGU9aTtPYmplY3QuZGVmaW5lUHJvcGVydHkoZSwiX19lc01vZHVsZSIse3ZhbHVlOiEwfSksZS5UZXJtaW5hbD12b2lkIDA7dmFyIHQ9cigzMjM2KSxuPXIoOTA0Miksbz1yKDc5NzUpLHM9cig3MDkwKSxhPXIoNTc0MSksYz1yKDgyODUpLGw9WyJjb2xzIiwicm93cyJdLHU9ZnVuY3Rpb24oKXtmdW5jdGlvbiBlKGUpe3ZhciByPXRoaXM7dGhpcy5fY29yZT1uZXcgdC5UZXJtaW5hbChlKSx0aGlzLl9hZGRvbk1hbmFnZXI9bmV3IGEuQWRkb25NYW5hZ2VyLHRoaXMuX3B1YmxpY09wdGlvbnM9e307dmFyIGk9ZnVuY3Rpb24oZSl7T2JqZWN0LmRlZmluZVByb3BlcnR5KG4uX3B1YmxpY09wdGlvbnMsZSx7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHIuX2NvcmUub3B0aW9uc1tlXX0sc2V0OmZ1bmN0aW9uKHQpe3IuX2NoZWNrUmVhZG9ubHlPcHRpb25zKGUpLHIuX2NvcmUub3B0aW9uc1tlXT10fX0pfSxuPXRoaXM7Zm9yKHZhciBvIGluIHRoaXMuX2NvcmUub3B0aW9ucylpKG8pfXJldHVybiBlLnByb3RvdHlwZS5fY2hlY2tSZWFkb25seU9wdGlvbnM9ZnVuY3Rpb24oZSl7aWYobC5pbmNsdWRlcyhlKSl0aHJvdyBuZXcgRXJyb3IoJ09wdGlvbiAiJytlKyciIGNhbiBvbmx5IGJlIHNldCBpbiB0aGUgY29uc3RydWN0b3InKX0sZS5wcm90b3R5cGUuX2NoZWNrUHJvcG9zZWRBcGk9ZnVuY3Rpb24oKXtpZighdGhpcy5fY29yZS5vcHRpb25zU2VydmljZS5vcHRpb25zLmFsbG93UHJvcG9zZWRBcGkpdGhyb3cgbmV3IEVycm9yKCJZb3UgbXVzdCBzZXQgdGhlIGFsbG93UHJvcG9zZWRBcGkgb3B0aW9uIHRvIHRydWUgdG8gdXNlIHByb3Bvc2VkIEFQSSIpfSxPYmplY3QuZGVmaW5lUHJvcGVydHkoZS5wcm90b3R5cGUsIm9uQmVsbCIse2dldDpmdW5jdGlvbigpe3JldHVybiB0aGlzLl9jb3JlLm9uQmVsbH0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxPYmplY3QuZGVmaW5lUHJvcGVydHkoZS5wcm90b3R5cGUsIm9uQmluYXJ5Iix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX2NvcmUub25CaW5hcnl9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KGUucHJvdG90eXBlLCJvbkN1cnNvck1vdmUiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fY29yZS5vbkN1cnNvck1vdmV9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KGUucHJvdG90eXBlLCJvbkRhdGEiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fY29yZS5vbkRhdGF9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KGUucHJvdG90eXBlLCJvbktleSIse2dldDpmdW5jdGlvbigpe3JldHVybiB0aGlzLl9jb3JlLm9uS2V5fSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLE9iamVjdC5kZWZpbmVQcm9wZXJ0eShlLnByb3RvdHlwZSwib25MaW5lRmVlZCIse2dldDpmdW5jdGlvbigpe3JldHVybiB0aGlzLl9jb3JlLm9uTGluZUZlZWR9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KGUucHJvdG90eXBlLCJvblJlbmRlciIse2dldDpmdW5jdGlvbigpe3JldHVybiB0aGlzLl9jb3JlLm9uUmVuZGVyfSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLE9iamVjdC5kZWZpbmVQcm9wZXJ0eShlLnByb3RvdHlwZSwib25SZXNpemUiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fY29yZS5vblJlc2l6ZX0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxPYmplY3QuZGVmaW5lUHJvcGVydHkoZS5wcm90b3R5cGUsIm9uU2Nyb2xsIix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX2NvcmUub25TY3JvbGx9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KGUucHJvdG90eXBlLCJvblNlbGVjdGlvbkNoYW5nZSIse2dldDpmdW5jdGlvbigpe3JldHVybiB0aGlzLl9jb3JlLm9uU2VsZWN0aW9uQ2hhbmdlfSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLE9iamVjdC5kZWZpbmVQcm9wZXJ0eShlLnByb3RvdHlwZSwib25UaXRsZUNoYW5nZSIse2dldDpmdW5jdGlvbigpe3JldHVybiB0aGlzLl9jb3JlLm9uVGl0bGVDaGFuZ2V9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KGUucHJvdG90eXBlLCJlbGVtZW50Iix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX2NvcmUuZWxlbWVudH0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxPYmplY3QuZGVmaW5lUHJvcGVydHkoZS5wcm90b3R5cGUsInBhcnNlciIse2dldDpmdW5jdGlvbigpe3JldHVybiB0aGlzLl9jaGVja1Byb3Bvc2VkQXBpKCksdGhpcy5fcGFyc2VyfHwodGhpcy5fcGFyc2VyPW5ldyBvLlBhcnNlckFwaSh0aGlzLl9jb3JlKSksdGhpcy5fcGFyc2VyfSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLE9iamVjdC5kZWZpbmVQcm9wZXJ0eShlLnByb3RvdHlwZSwidW5pY29kZSIse2dldDpmdW5jdGlvbigpe3JldHVybiB0aGlzLl9jaGVja1Byb3Bvc2VkQXBpKCksbmV3IHMuVW5pY29kZUFwaSh0aGlzLl9jb3JlKX0sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxPYmplY3QuZGVmaW5lUHJvcGVydHkoZS5wcm90b3R5cGUsInRleHRhcmVhIix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX2NvcmUudGV4dGFyZWF9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KGUucHJvdG90eXBlLCJyb3dzIix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX2NvcmUucm93c30sZW51bWVyYWJsZTohMSxjb25maWd1cmFibGU6ITB9KSxPYmplY3QuZGVmaW5lUHJvcGVydHkoZS5wcm90b3R5cGUsImNvbHMiLHtnZXQ6ZnVuY3Rpb24oKXtyZXR1cm4gdGhpcy5fY29yZS5jb2xzfSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLE9iamVjdC5kZWZpbmVQcm9wZXJ0eShlLnByb3RvdHlwZSwiYnVmZmVyIix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX2NoZWNrUHJvcG9zZWRBcGkoKSx0aGlzLl9idWZmZXJ8fCh0aGlzLl9idWZmZXI9bmV3IGMuQnVmZmVyTmFtZXNwYWNlQXBpKHRoaXMuX2NvcmUpKSx0aGlzLl9idWZmZXJ9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KGUucHJvdG90eXBlLCJtYXJrZXJzIix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIHRoaXMuX2NoZWNrUHJvcG9zZWRBcGkoKSx0aGlzLl9jb3JlLm1hcmtlcnN9LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksT2JqZWN0LmRlZmluZVByb3BlcnR5KGUucHJvdG90eXBlLCJtb2RlcyIse2dldDpmdW5jdGlvbigpe3ZhciBlPXRoaXMuX2NvcmUuY29yZVNlcnZpY2UuZGVjUHJpdmF0ZU1vZGVzLHQ9Im5vbmUiO3N3aXRjaCh0aGlzLl9jb3JlLmNvcmVNb3VzZVNlcnZpY2UuYWN0aXZlUHJvdG9jb2wpe2Nhc2UiWDEwIjp0PSJ4MTAiO2JyZWFrO2Nhc2UiVlQyMDAiOnQ9InZ0MjAwIjticmVhaztjYXNlIkRSQUciOnQ9ImRyYWciO2JyZWFrO2Nhc2UiQU5ZIjp0PSJhbnkifXJldHVybnthcHBsaWNhdGlvbkN1cnNvcktleXNNb2RlOmUuYXBwbGljYXRpb25DdXJzb3JLZXlzLGFwcGxpY2F0aW9uS2V5cGFkTW9kZTplLmFwcGxpY2F0aW9uS2V5cGFkLGJyYWNrZXRlZFBhc3RlTW9kZTplLmJyYWNrZXRlZFBhc3RlTW9kZSxpbnNlcnRNb2RlOnRoaXMuX2NvcmUuY29yZVNlcnZpY2UubW9kZXMuaW5zZXJ0TW9kZSxtb3VzZVRyYWNraW5nTW9kZTp0LG9yaWdpbk1vZGU6ZS5vcmlnaW4scmV2ZXJzZVdyYXBhcm91bmRNb2RlOmUucmV2ZXJzZVdyYXBhcm91bmQsc2VuZEZvY3VzTW9kZTplLnNlbmRGb2N1cyx3cmFwYXJvdW5kTW9kZTplLndyYXBhcm91bmR9fSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLE9iamVjdC5kZWZpbmVQcm9wZXJ0eShlLnByb3RvdHlwZSwib3B0aW9ucyIse2dldDpmdW5jdGlvbigpe3JldHVybiB0aGlzLl9wdWJsaWNPcHRpb25zfSxzZXQ6ZnVuY3Rpb24oZSl7Zm9yKHZhciB0IGluIGUpdGhpcy5fcHVibGljT3B0aW9uc1t0XT1lW3RdfSxlbnVtZXJhYmxlOiExLGNvbmZpZ3VyYWJsZTohMH0pLGUucHJvdG90eXBlLmJsdXI9ZnVuY3Rpb24oKXt0aGlzLl9jb3JlLmJsdXIoKX0sZS5wcm90b3R5cGUuZm9jdXM9ZnVuY3Rpb24oKXt0aGlzLl9jb3JlLmZvY3VzKCl9LGUucHJvdG90eXBlLnJlc2l6ZT1mdW5jdGlvbihlLHQpe3RoaXMuX3ZlcmlmeUludGVnZXJzKGUsdCksdGhpcy5fY29yZS5yZXNpemUoZSx0KX0sZS5wcm90b3R5cGUub3Blbj1mdW5jdGlvbihlKXt0aGlzLl9jb3JlLm9wZW4oZSl9LGUucHJvdG90eXBlLmF0dGFjaEN1c3RvbUtleUV2ZW50SGFuZGxlcj1mdW5jdGlvbihlKXt0aGlzLl9jb3JlLmF0dGFjaEN1c3RvbUtleUV2ZW50SGFuZGxlcihlKX0sZS5wcm90b3R5cGUucmVnaXN0ZXJMaW5rTWF0Y2hlcj1mdW5jdGlvbihlLHQscil7cmV0dXJuIHRoaXMuX2NoZWNrUHJvcG9zZWRBcGkoKSx0aGlzLl9jb3JlLnJlZ2lzdGVyTGlua01hdGNoZXIoZSx0LHIpfSxlLnByb3RvdHlwZS5kZXJlZ2lzdGVyTGlua01hdGNoZXI9ZnVuY3Rpb24oZSl7dGhpcy5fY2hlY2tQcm9wb3NlZEFwaSgpLHRoaXMuX2NvcmUuZGVyZWdpc3RlckxpbmtNYXRjaGVyKGUpfSxlLnByb3RvdHlwZS5yZWdpc3RlckxpbmtQcm92aWRlcj1mdW5jdGlvbihlKXtyZXR1cm4gdGhpcy5fY2hlY2tQcm9wb3NlZEFwaSgpLHRoaXMuX2NvcmUucmVnaXN0ZXJMaW5rUHJvdmlkZXIoZSl9LGUucHJvdG90eXBlLnJlZ2lzdGVyQ2hhcmFjdGVySm9pbmVyPWZ1bmN0aW9uKGUpe3JldHVybiB0aGlzLl9jaGVja1Byb3Bvc2VkQXBpKCksdGhpcy5fY29yZS5yZWdpc3RlckNoYXJhY3RlckpvaW5lcihlKX0sZS5wcm90b3R5cGUuZGVyZWdpc3RlckNoYXJhY3RlckpvaW5lcj1mdW5jdGlvbihlKXt0aGlzLl9jaGVja1Byb3Bvc2VkQXBpKCksdGhpcy5fY29yZS5kZXJlZ2lzdGVyQ2hhcmFjdGVySm9pbmVyKGUpfSxlLnByb3RvdHlwZS5yZWdpc3Rlck1hcmtlcj1mdW5jdGlvbihlKXtyZXR1cm4gdGhpcy5fY2hlY2tQcm9wb3NlZEFwaSgpLHRoaXMuX3ZlcmlmeUludGVnZXJzKGUpLHRoaXMuX2NvcmUuYWRkTWFya2VyKGUpfSxlLnByb3RvdHlwZS5hZGRNYXJrZXI9ZnVuY3Rpb24oZSl7cmV0dXJuIHRoaXMucmVnaXN0ZXJNYXJrZXIoZSl9LGUucHJvdG90eXBlLmhhc1NlbGVjdGlvbj1mdW5jdGlvbigpe3JldHVybiB0aGlzLl9jb3JlLmhhc1NlbGVjdGlvbigpfSxlLnByb3RvdHlwZS5zZWxlY3Q9ZnVuY3Rpb24oZSx0LHIpe3RoaXMuX3ZlcmlmeUludGVnZXJzKGUsdCxyKSx0aGlzLl9jb3JlLnNlbGVjdChlLHQscil9LGUucHJvdG90eXBlLmdldFNlbGVjdGlvbj1mdW5jdGlvbigpe3JldHVybiB0aGlzLl9jb3JlLmdldFNlbGVjdGlvbigpfSxlLnByb3RvdHlwZS5nZXRTZWxlY3Rpb25Qb3NpdGlvbj1mdW5jdGlvbigpe3JldHVybiB0aGlzLl9jb3JlLmdldFNlbGVjdGlvblBvc2l0aW9uKCl9LGUucHJvdG90eXBlLmNsZWFyU2VsZWN0aW9uPWZ1bmN0aW9uKCl7dGhpcy5fY29yZS5jbGVhclNlbGVjdGlvbigpfSxlLnByb3RvdHlwZS5zZWxlY3RBbGw9ZnVuY3Rpb24oKXt0aGlzLl9jb3JlLnNlbGVjdEFsbCgpfSxlLnByb3RvdHlwZS5zZWxlY3RMaW5lcz1mdW5jdGlvbihlLHQpe3RoaXMuX3ZlcmlmeUludGVnZXJzKGUsdCksdGhpcy5fY29yZS5zZWxlY3RMaW5lcyhlLHQpfSxlLnByb3RvdHlwZS5kaXNwb3NlPWZ1bmN0aW9uKCl7dGhpcy5fYWRkb25NYW5hZ2VyLmRpc3Bvc2UoKSx0aGlzLl9jb3JlLmRpc3Bvc2UoKX0sZS5wcm90b3R5cGUuc2Nyb2xsTGluZXM9ZnVuY3Rpb24oZSl7dGhpcy5fdmVyaWZ5SW50ZWdlcnMoZSksdGhpcy5fY29yZS5zY3JvbGxMaW5lcyhlKX0sZS5wcm90b3R5cGUuc2Nyb2xsUGFnZXM9ZnVuY3Rpb24oZSl7dGhpcy5fdmVyaWZ5SW50ZWdlcnMoZSksdGhpcy5fY29yZS5zY3JvbGxQYWdlcyhlKX0sZS5wcm90b3R5cGUuc2Nyb2xsVG9Ub3A9ZnVuY3Rpb24oKXt0aGlzLl9jb3JlLnNjcm9sbFRvVG9wKCl9LGUucHJvdG90eXBlLnNjcm9sbFRvQm90dG9tPWZ1bmN0aW9uKCl7dGhpcy5fY29yZS5zY3JvbGxUb0JvdHRvbSgpfSxlLnByb3RvdHlwZS5zY3JvbGxUb0xpbmU9ZnVuY3Rpb24oZSl7dGhpcy5fdmVyaWZ5SW50ZWdlcnMoZSksdGhpcy5fY29yZS5zY3JvbGxUb0xpbmUoZSl9LGUucHJvdG90eXBlLmNsZWFyPWZ1bmN0aW9uKCl7dGhpcy5fY29yZS5jbGVhcigpfSxlLnByb3RvdHlwZS53cml0ZT1mdW5jdGlvbihlLHQpe3RoaXMuX2NvcmUud3JpdGUoZSx0KX0sZS5wcm90b3R5cGUud3JpdGVVdGY4PWZ1bmN0aW9uKGUsdCl7dGhpcy5fY29yZS53cml0ZShlLHQpfSxlLnByb3RvdHlwZS53cml0ZWxuPWZ1bmN0aW9uKGUsdCl7dGhpcy5fY29yZS53cml0ZShlKSx0aGlzLl9jb3JlLndyaXRlKCJcclxuIix0KX0sZS5wcm90b3R5cGUucGFzdGU9ZnVuY3Rpb24oZSl7dGhpcy5fY29yZS5wYXN0ZShlKX0sZS5wcm90b3R5cGUuZ2V0T3B0aW9uPWZ1bmN0aW9uKGUpe3JldHVybiB0aGlzLl9jb3JlLm9wdGlvbnNTZXJ2aWNlLmdldE9wdGlvbihlKX0sZS5wcm90b3R5cGUuc2V0T3B0aW9uPWZ1bmN0aW9uKGUsdCl7dGhpcy5fY2hlY2tSZWFkb25seU9wdGlvbnMoZSksdGhpcy5fY29yZS5vcHRpb25zU2VydmljZS5zZXRPcHRpb24oZSx0KX0sZS5wcm90b3R5cGUucmVmcmVzaD1mdW5jdGlvbihlLHQpe3RoaXMuX3ZlcmlmeUludGVnZXJzKGUsdCksdGhpcy5fY29yZS5yZWZyZXNoKGUsdCl9LGUucHJvdG90eXBlLnJlc2V0PWZ1bmN0aW9uKCl7dGhpcy5fY29yZS5yZXNldCgpfSxlLnByb3RvdHlwZS5jbGVhclRleHR1cmVBdGxhcz1mdW5jdGlvbigpe3RoaXMuX2NvcmUuY2xlYXJUZXh0dXJlQXRsYXMoKX0sZS5wcm90b3R5cGUubG9hZEFkZG9uPWZ1bmN0aW9uKGUpe3JldHVybiB0aGlzLl9hZGRvbk1hbmFnZXIubG9hZEFkZG9uKHRoaXMsZSl9LE9iamVjdC5kZWZpbmVQcm9wZXJ0eShlLCJzdHJpbmdzIix7Z2V0OmZ1bmN0aW9uKCl7cmV0dXJuIG59LGVudW1lcmFibGU6ITEsY29uZmlndXJhYmxlOiEwfSksZS5wcm90b3R5cGUuX3ZlcmlmeUludGVnZXJzPWZ1bmN0aW9uKCl7Zm9yKHZhciBlPVtdLHQ9MDt0PGFyZ3VtZW50cy5sZW5ndGg7dCsrKWVbdF09YXJndW1lbnRzW3RdO2Zvcih2YXIgcj0wLGk9ZTtyPGkubGVuZ3RoO3IrKyl7dmFyIG49aVtyXTtpZihuPT09MS8wfHxpc05hTihuKXx8biUxIT0wKXRocm93IG5ldyBFcnJvcigiVGhpcyBBUEkgb25seSBhY2NlcHRzIGludGVnZXJzIil9fSxlfSgpO2UuVGVybWluYWw9dX0pKCksaX0pKCl9fSx0PXt9O2Z1bmN0aW9uIHIoaSl7dmFyIG49dFtpXTtpZih2b2lkIDAhPT1uKXJldHVybiBuLmV4cG9ydHM7dmFyIG89dFtpXT17aWQ6aSxsb2FkZWQ6ITEsZXhwb3J0czp7fX07cmV0dXJuIGVbaV0uY2FsbChvLmV4cG9ydHMsbyxvLmV4cG9ydHMsciksby5sb2FkZWQ9ITAsby5leHBvcnRzfXIubj1lPT57dmFyIHQ9ZSYmZS5fX2VzTW9kdWxlPygpPT5lLmRlZmF1bHQ6KCk9PmU7cmV0dXJuIHIuZCh0LHthOnR9KSx0fSxyLmQ9KGUsdCk9Pntmb3IodmFyIGkgaW4gdClyLm8odCxpKSYmIXIubyhlLGkpJiZPYmplY3QuZGVmaW5lUHJvcGVydHkoZSxpLHtlbnVtZXJhYmxlOiEwLGdldDp0W2ldfSl9LHIuZz1mdW5jdGlvbigpe2lmKCJvYmplY3QiPT10eXBlb2YgZ2xvYmFsVGhpcylyZXR1cm4gZ2xvYmFsVGhpczt0cnl7cmV0dXJuIHRoaXN8fG5ldyBGdW5jdGlvbigicmV0dXJuIHRoaXMiKSgpfWNhdGNoKGUpe2lmKCJvYmplY3QiPT10eXBlb2Ygd2luZG93KXJldHVybiB3aW5kb3d9fSgpLHIubz0oZSx0KT0+T2JqZWN0LnByb3RvdHlwZS5oYXNPd25Qcm9wZXJ0eS5jYWxsKGUsdCksci5ubWQ9ZT0+KGUucGF0aHM9W10sZS5jaGlsZHJlbnx8KGUuY2hpbGRyZW49W10pLGUpLCgoKT0+eyJ1c2Ugc3RyaWN0Ijt2YXIgZT1yKDM3OSksdD1yLm4oZSksaT1yKDc5NSksbj1yLm4oaSksbz1yKDU2OSkscz1yLm4obyksYT1yKDU2NSksYz1yLm4oYSksbD1yKDIxNiksdT1yLm4obCksaD1yKDU4OSksZj1yLm4oaCksXz1yKDEwMiksZD17fTtkLnN0eWxlVGFnVHJhbnNmb3JtPWYoKSxkLnNldEF0dHJpYnV0ZXM9YygpLGQuaW5zZXJ0PXMoKS5iaW5kKG51bGwsImhlYWQiKSxkLmRvbUFQST1uKCksZC5pbnNlcnRTdHlsZUVsZW1lbnQ9dSgpLHQoKShfLlosZCksXy5aJiZfLloubG9jYWxzJiZfLloubG9jYWxzO3ZhciBwPXIoMzIwKSx2PXIoNjE3KSxnPXIoNDg2KSx5PXIubihnKSxtPWZ1bmN0aW9uKGUsdCxyLGkpe3JldHVybiBuZXcocnx8KHI9UHJvbWlzZSkpKChmdW5jdGlvbihuLG8pe2Z1bmN0aW9uIHMoZSl7dHJ5e2MoaS5uZXh0KGUpKX1jYXRjaChlKXtvKGUpfX1mdW5jdGlvbiBhKGUpe3RyeXtjKGkudGhyb3coZSkpfWNhdGNoKGUpe28oZSl9fWZ1bmN0aW9uIGMoZSl7dmFyIHQ7ZS5kb25lP24oZS52YWx1ZSk6KHQ9ZS52YWx1ZSx0IGluc3RhbmNlb2Ygcj90Om5ldyByKChmdW5jdGlvbihlKXtlKHQpfSkpKS50aGVuKHMsYSl9YygoaT1pLmFwcGx5KGUsdHx8W10pKS5uZXh0KCkpfSkpfSxiPWZ1bmN0aW9uKGUsdCl7dmFyIHIsaSxuLG8scz17bGFiZWw6MCxzZW50OmZ1bmN0aW9uKCl7aWYoMSZuWzBdKXRocm93IG5bMV07cmV0dXJuIG5bMV19LHRyeXM6W10sb3BzOltdfTtyZXR1cm4gbz17bmV4dDphKDApLHRocm93OmEoMSkscmV0dXJuOmEoMil9LCJmdW5jdGlvbiI9PXR5cGVvZiBTeW1ib2wmJihvW1N5bWJvbC5pdGVyYXRvcl09ZnVuY3Rpb24oKXtyZXR1cm4gdGhpc30pLG87ZnVuY3Rpb24gYShvKXtyZXR1cm4gZnVuY3Rpb24oYSl7cmV0dXJuIGZ1bmN0aW9uKG8pe2lmKHIpdGhyb3cgbmV3IFR5cGVFcnJvcigiR2VuZXJhdG9yIGlzIGFscmVhZHkgZXhlY3V0aW5nLiIpO2Zvcig7czspdHJ5e2lmKHI9MSxpJiYobj0yJm9bMF0/aS5yZXR1cm46b1swXT9pLnRocm93fHwoKG49aS5yZXR1cm4pJiZuLmNhbGwoaSksMCk6aS5uZXh0KSYmIShuPW4uY2FsbChpLG9bMV0pKS5kb25lKXJldHVybiBuO3N3aXRjaChpPTAsbiYmKG89WzImb1swXSxuLnZhbHVlXSksb1swXSl7Y2FzZSAwOmNhc2UgMTpuPW87YnJlYWs7Y2FzZSA0OnJldHVybiBzLmxhYmVsKysse3ZhbHVlOm9bMV0sZG9uZTohMX07Y2FzZSA1OnMubGFiZWwrKyxpPW9bMV0sbz1bMF07Y29udGludWU7Y2FzZSA3Om89cy5vcHMucG9wKCkscy50cnlzLnBvcCgpO2NvbnRpbnVlO2RlZmF1bHQ6aWYoISgobj0obj1zLnRyeXMpLmxlbmd0aD4wJiZuW24ubGVuZ3RoLTFdKXx8NiE9PW9bMF0mJjIhPT1vWzBdKSl7cz0wO2NvbnRpbnVlfWlmKDM9PT1vWzBdJiYoIW58fG9bMV0+blswXSYmb1sxXTxuWzNdKSl7cy5sYWJlbD1vWzFdO2JyZWFrfWlmKDY9PT1vWzBdJiZzLmxhYmVsPG5bMV0pe3MubGFiZWw9blsxXSxuPW87YnJlYWt9aWYobiYmcy5sYWJlbDxuWzJdKXtzLmxhYmVsPW5bMl0scy5vcHMucHVzaChvKTticmVha31uWzJdJiZzLm9wcy5wb3AoKSxzLnRyeXMucG9wKCk7Y29udGludWV9bz10LmNhbGwoZSxzKX1jYXRjaChlKXtvPVs2LGVdLGk9MH1maW5hbGx5e3I9bj0wfWlmKDUmb1swXSl0aHJvdyBvWzFdO3JldHVybnt2YWx1ZTpvWzBdP29bMV06dm9pZCAwLGRvbmU6ITB9fShbbyxhXSl9fX07d2luZG93Lm9ubG9hZD1mdW5jdGlvbigpe3ZhciBlPW5ldyBwLlRlcm1pbmFsLHQ9bmV3IHYuRml0QWRkb247d2luZG93LnRlcm09ZSx3aW5kb3cuZml0QWRkb249dCxlLmxvYWRBZGRvbih0KSxlLm9wZW4oZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoInRlcm1pbmFsIikpO3ZhciByPWZ1bmN0aW9uKCl7ZS5lbGVtZW50LnBhcmVudEVsZW1lbnQuc3R5bGUuaGVpZ2h0PXdpbmRvdy5pbm5lckhlaWdodC0xNisicHgiLHQuZml0KCksZmV0Y2goIi9yZXNpemU/cm93cz0iK2Uucm93cysiJmNvbHM9IitlLmNvbHMpfTtyKCksd2luZG93Lm9ucmVzaXplPXI7dmFyIGk9W107ZS5vbkRhdGEoKGZ1bmN0aW9uKGUpe2kucHVzaChlKX0pKSxtKHRoaXMsdm9pZCAwLHZvaWQgMCwoZnVuY3Rpb24oKXt2YXIgZSx0LHI7cmV0dXJuIGIodGhpcywoZnVuY3Rpb24obil7c3dpdGNoKG4ubGFiZWwpe2Nhc2UgMDplPWZ1bmN0aW9uKGUpe3JldHVybiBuZXcgUHJvbWlzZSgoZnVuY3Rpb24odCl7cmV0dXJuIHNldFRpbWVvdXQodCxlKX0pKX0sbi5sYWJlbD0xO2Nhc2UgMTpuLnRyeXMucHVzaChbMSwsNyw4XSksbi5sYWJlbD0yO2Nhc2UgMjpyZXR1cm5bNCxlKDEwMCldO2Nhc2UgMzpyZXR1cm4gbi5zZW50KCkseSgpLmlzRW1wdHkoaSk/WzMsNV06KHQ9aS5qb2luKCIiKSxyPXdpbmRvdy5idG9hKHQpLGkubGVuZ3RoPTAsWzQsZmV0Y2goIi9pbi8iK3IpXSk7Y2FzZSA0Om4uc2VudCgpLG4ubGFiZWw9NTtjYXNlIDU6cmV0dXJuWzMsMl07Y2FzZSA2OnJldHVyblszLDhdO2Nhc2UgNzpyZXR1cm4gY29uc29sZS5sb2coImlucHV0IGRpc2Nvbm5lY3QhIiksWzddO2Nhc2UgODpyZXR1cm5bMl19fSkpfSkpLGZ1bmN0aW9uKCl7bSh0aGlzLHZvaWQgMCx2b2lkIDAsKGZ1bmN0aW9uKCl7dmFyIHQscixpO3JldHVybiBiKHRoaXMsKGZ1bmN0aW9uKG4pe3N3aXRjaChuLmxhYmVsKXtjYXNlIDA6bi50cnlzLnB1c2goWzAsLDUsNl0pLG4ubGFiZWw9MTtjYXNlIDE6cmV0dXJuWzQsZmV0Y2goIi9vdXQiKV07Y2FzZSAyOnJldHVybiB0PW4uc2VudCgpLGk9VWludDhBcnJheS5iaW5kLFs0LHQuYXJyYXlCdWZmZXIoKV07Y2FzZSAzOnJldHVybiByPW5ldyhpLmFwcGx5KFVpbnQ4QXJyYXksW3ZvaWQgMCxuLnNlbnQoKV0pKSx0JiZlLndyaXRlKHIpLFszLDFdO2Nhc2UgNDpyZXR1cm5bMyw2XTtjYXNlIDU6cmV0dXJuIGNvbnNvbGUubG9nKCJpbnB1dCBkaXNjb25uZWN0ISIpLFs3XTtjYXNlIDY6cmV0dXJuWzJdfX0pKX0pKX0oKX19KSgpfSkoKTs=", + "headers": [ + [ + "content-length", + "426644" + ], + [ + "content-type", + "text/javascript" + ] + ], + "ok": true, + "status": 200, + "status_text": "" + }, + "https://localhost:10000/out": { + "data": "W3N1cGVyZ2F0ZXdheV0gUE9TVCAvbWVzc2FnZSAtPiBTU0UgdHJhbnNwb3J0DQpbc3VwZXJnYXRld2F5XSBTU0UgLT4gQ2hpbGQ6IHsianNvbnJwYyI6IjIuMCIsImlkIjowLCJtZXRob2QiOiJpbml0aWFsaXplIiwicGFyYW1zIjp7InByb3RvY29sVmVyc2lvbiI6IjIwMjQtMTEtMDUiLCJjYXBhYmlsaXRpZXMiOnsicm9vdHMiOnsibGlzdENoYW5nZWQiOnRydWV9fSwiY2xpZW50SW5mbyI6eyJuYW1lIjoibWNwIiwidmVyc2lvbiI6IjAuMS4wIn19fQ0KW3N1cGVyZ2F0ZXdheV0gQ2hpbGQgLT4gU1NFOiB7DQogIHJlc3VsdDogew0KICAgIHByb3RvY29sVmVyc2lvbjogG1szMm0nMjAyNC0xMS0wNScbWzM5bSwNCiAgICBjYXBhYmlsaXRpZXM6IHsgdG9vbHM6IHt9IH0sDQogICAgc2VydmVySW5mbzogeyBuYW1lOiAbWzMybSdzZWN1cmUtZmlsZXN5c3RlbS1zZXJ2ZXInG1szOW0sIHZlcnNpb246IBtbMzJtJzAuMi4wJxtbMzltIH0NCiAgfSwNCiAganNvbnJwYzogG1szMm0nMi4wJxtbMzltLA0KICBpZDogG1szM20wG1szOW0NCn0NCltzdXBlcmdhdGV3YXldIFBPU1QgL21lc3NhZ2UgLT4gU1NFIHRyYW5zcG9ydA0KW3N1cGVyZ2F0ZXdheV0gU1NFIC0+IENoaWxkOiB7Impzb25ycGMiOiIyLjAiLCJtZXRob2QiOiJub3RpZmljYXRpb25zL2luaXRpYWxpemVkIn0NCltzdXBlcmdhdGV3YXldIFBPU1QgL21lc3NhZ2UgLT4gU1NFIHRyYW5zcG9ydA0KW3N1cGVyZ2F0ZXdheV0gU1NFIC0+IENoaWxkOiB7Impzb25ycGMiOiIyLjAiLCJpZCI6MSwibWV0aG9kIjoidG9vbHMvY2FsbCIsInBhcmFtcyI6eyJuYW1lIjoibGlzdF9kaXJlY3RvcnkiLCJhcmd1bWVudHMiOnsic2Vzc2lvbl9pZCI6IjI1ZmU0OWQwLTg4YzAtNGQ3OC05MDFhLWI3YmQyMTBhNGQ1MiIsInBhdGgiOiIvY29udGVudCJ9fX0NCltzdXBlcmdhdGV3YXldIENoaWxkIC0+IFNTRTogeyByZXN1bHQ6IHsgY29udGVudDogWyAbWzM2bVtPYmplY3RdG1szOW0gXSB9LCBqc29ucnBjOiAbWzMybScyLjAnG1szOW0sIGlkOiAbWzMzbTEbWzM5bSB9DQpbc3VwZXJnYXRld2F5XSBTU0UgY29ubmVjdGlvbiBjbG9zZWQuDQo=", + "headers": [ + [ + "content-length", + "1067" + ], + [ + "content-type", + "text/html; charset=UTF-8" + ] + ], + "ok": true, + "status": 200, + "status_text": "" + }, + "https://localhost:10000/resize?rows=46&cols=196": { + "data": "", + "headers": [ + [ + "content-length", + "0" + ], + [ + "content-type", + "text/html; charset=UTF-8" + ] + ], + "ok": true, + "status": 200, + "status_text": "" + } + } + }, + "id": "giIA2M-ANUIM", + "outputId": "612c3487-1fd7-41ab-f65a-690b1325f46d" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "Launching Xterm..." + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": "\n (async () => {\n const url = new URL(await google.colab.kernel.proxyPort(10000, {'cache': true}));\n const iframe = document.createElement('iframe');\n iframe.src = url;\n iframe.setAttribute('width', '100%');\n iframe.setAttribute('height', '800');\n iframe.setAttribute('frameborder', 0);\n document.body.appendChild(iframe);\n })();\n ", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "\n", + "%xterm\n", + "# touch /content/foo\n", + "# touch /content/bar\n", + "# npx -y supergateway --port 8000 --stdio 'npx -y @modelcontextprotocol/server-filesystem /content'" + ] + }, + { + "cell_type": "markdown", + "id": "f4ksBP6MN7cB", + "metadata": { + "id": "f4ksBP6MN7cB" + }, + "source": [ + "Register the toolgroup hosted in the MCP server with llama stack and verify if the stack discovers the tools correctly" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "DwdKhQb1N295", + "metadata": { + "id": "DwdKhQb1N295" + }, + "outputs": [], + "source": [ + "from llama_stack_client.types.shared_params.url import URL\n", + "client.toolgroups.register(\n", + " toolgroup_id=\"mcp::filesystem\",\n", + " provider_id=\"model-context-protocol\",\n", + " mcp_endpoint=URL(uri=\"http://localhost:8000/sse\"),\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "ZZ5_vIkDOyAN", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "id": "ZZ5_vIkDOyAN", + "outputId": "f6fa8639-c2d8-497d-f4ed-716b3bf775d4" + }, + "outputs": [ + { + "data": { + "text/html": [ + "
[\n",
+              "│   Tool(\n",
+              "│   │   description='Read the complete contents of a file from the file system. Handles various text encodings and provides detailed error messages if the file cannot be read. Use this tool when you need to examine the contents of a single file. Only works within allowed directories.',\n",
+              "│   │   identifier='read_file',\n",
+              "│   │   parameters=[Parameter(description='', name='path', parameter_type='string', required=True, default=None)],\n",
+              "│   │   provider_id='model-context-protocol',\n",
+              "│   │   provider_resource_id='read_file',\n",
+              "│   │   tool_host='model_context_protocol',\n",
+              "│   │   toolgroup_id='mcp::filesystem',\n",
+              "│   │   type='tool',\n",
+              "│   │   metadata={'endpoint': 'http://localhost:8000/sse'}\n",
+              "│   ),\n",
+              "│   Tool(\n",
+              "│   │   description=\"Read the contents of multiple files simultaneously. This is more efficient than reading files one by one when you need to analyze or compare multiple files. Each file's content is returned with its path as a reference. Failed reads for individual files won't stop the entire operation. Only works within allowed directories.\",\n",
+              "│   │   identifier='read_multiple_files',\n",
+              "│   │   parameters=[Parameter(description='', name='paths', parameter_type='array', required=True, default=None)],\n",
+              "│   │   provider_id='model-context-protocol',\n",
+              "│   │   provider_resource_id='read_multiple_files',\n",
+              "│   │   tool_host='model_context_protocol',\n",
+              "│   │   toolgroup_id='mcp::filesystem',\n",
+              "│   │   type='tool',\n",
+              "│   │   metadata={'endpoint': 'http://localhost:8000/sse'}\n",
+              "│   ),\n",
+              "│   Tool(\n",
+              "│   │   description='Create a new file or completely overwrite an existing file with new content. Use with caution as it will overwrite existing files without warning. Handles text content with proper encoding. Only works within allowed directories.',\n",
+              "│   │   identifier='write_file',\n",
+              "│   │   parameters=[\n",
+              "│   │   │   Parameter(description='', name='path', parameter_type='string', required=True, default=None),\n",
+              "│   │   │   Parameter(description='', name='content', parameter_type='string', required=True, default=None)\n",
+              "│   │   ],\n",
+              "│   │   provider_id='model-context-protocol',\n",
+              "│   │   provider_resource_id='write_file',\n",
+              "│   │   tool_host='model_context_protocol',\n",
+              "│   │   toolgroup_id='mcp::filesystem',\n",
+              "│   │   type='tool',\n",
+              "│   │   metadata={'endpoint': 'http://localhost:8000/sse'}\n",
+              "│   ),\n",
+              "│   Tool(\n",
+              "│   │   description='Make line-based edits to a text file. Each edit replaces exact line sequences with new content. Returns a git-style diff showing the changes made. Only works within allowed directories.',\n",
+              "│   │   identifier='edit_file',\n",
+              "│   │   parameters=[\n",
+              "│   │   │   Parameter(description='', name='path', parameter_type='string', required=True, default=None),\n",
+              "│   │   │   Parameter(description='', name='edits', parameter_type='array', required=True, default=None),\n",
+              "│   │   │   Parameter(\n",
+              "│   │   │   │   description='Preview changes using git-style diff format',\n",
+              "│   │   │   │   name='dryRun',\n",
+              "│   │   │   │   parameter_type='boolean',\n",
+              "│   │   │   │   required=True,\n",
+              "│   │   │   │   default=None\n",
+              "│   │   │   )\n",
+              "│   │   ],\n",
+              "│   │   provider_id='model-context-protocol',\n",
+              "│   │   provider_resource_id='edit_file',\n",
+              "│   │   tool_host='model_context_protocol',\n",
+              "│   │   toolgroup_id='mcp::filesystem',\n",
+              "│   │   type='tool',\n",
+              "│   │   metadata={'endpoint': 'http://localhost:8000/sse'}\n",
+              "│   ),\n",
+              "│   Tool(\n",
+              "│   │   description='Create a new directory or ensure a directory exists. Can create multiple nested directories in one operation. If the directory already exists, this operation will succeed silently. Perfect for setting up directory structures for projects or ensuring required paths exist. Only works within allowed directories.',\n",
+              "│   │   identifier='create_directory',\n",
+              "│   │   parameters=[Parameter(description='', name='path', parameter_type='string', required=True, default=None)],\n",
+              "│   │   provider_id='model-context-protocol',\n",
+              "│   │   provider_resource_id='create_directory',\n",
+              "│   │   tool_host='model_context_protocol',\n",
+              "│   │   toolgroup_id='mcp::filesystem',\n",
+              "│   │   type='tool',\n",
+              "│   │   metadata={'endpoint': 'http://localhost:8000/sse'}\n",
+              "│   ),\n",
+              "│   Tool(\n",
+              "│   │   description='Get a detailed listing of all files and directories in a specified path. Results clearly distinguish between files and directories with [FILE] and [DIR] prefixes. This tool is essential for understanding directory structure and finding specific files within a directory. Only works within allowed directories.',\n",
+              "│   │   identifier='list_directory',\n",
+              "│   │   parameters=[Parameter(description='', name='path', parameter_type='string', required=True, default=None)],\n",
+              "│   │   provider_id='model-context-protocol',\n",
+              "│   │   provider_resource_id='list_directory',\n",
+              "│   │   tool_host='model_context_protocol',\n",
+              "│   │   toolgroup_id='mcp::filesystem',\n",
+              "│   │   type='tool',\n",
+              "│   │   metadata={'endpoint': 'http://localhost:8000/sse'}\n",
+              "│   ),\n",
+              "│   Tool(\n",
+              "│   │   description=\"Get a recursive tree view of files and directories as a JSON structure. Each entry includes 'name', 'type' (file/directory), and 'children' for directories. Files have no children array, while directories always have a children array (which may be empty). The output is formatted with 2-space indentation for readability. Only works within allowed directories.\",\n",
+              "│   │   identifier='directory_tree',\n",
+              "│   │   parameters=[Parameter(description='', name='path', parameter_type='string', required=True, default=None)],\n",
+              "│   │   provider_id='model-context-protocol',\n",
+              "│   │   provider_resource_id='directory_tree',\n",
+              "│   │   tool_host='model_context_protocol',\n",
+              "│   │   toolgroup_id='mcp::filesystem',\n",
+              "│   │   type='tool',\n",
+              "│   │   metadata={'endpoint': 'http://localhost:8000/sse'}\n",
+              "│   ),\n",
+              "│   Tool(\n",
+              "│   │   description='Move or rename files and directories. Can move files between directories and rename them in a single operation. If the destination exists, the operation will fail. Works across different directories and can be used for simple renaming within the same directory. Both source and destination must be within allowed directories.',\n",
+              "│   │   identifier='move_file',\n",
+              "│   │   parameters=[\n",
+              "│   │   │   Parameter(description='', name='source', parameter_type='string', required=True, default=None),\n",
+              "│   │   │   Parameter(description='', name='destination', parameter_type='string', required=True, default=None)\n",
+              "│   │   ],\n",
+              "│   │   provider_id='model-context-protocol',\n",
+              "│   │   provider_resource_id='move_file',\n",
+              "│   │   tool_host='model_context_protocol',\n",
+              "│   │   toolgroup_id='mcp::filesystem',\n",
+              "│   │   type='tool',\n",
+              "│   │   metadata={'endpoint': 'http://localhost:8000/sse'}\n",
+              "│   ),\n",
+              "│   Tool(\n",
+              "│   │   description=\"Recursively search for files and directories matching a pattern. Searches through all subdirectories from the starting path. The search is case-insensitive and matches partial names. Returns full paths to all matching items. Great for finding files when you don't know their exact location. Only searches within allowed directories.\",\n",
+              "│   │   identifier='search_files',\n",
+              "│   │   parameters=[\n",
+              "│   │   │   Parameter(description='', name='path', parameter_type='string', required=True, default=None),\n",
+              "│   │   │   Parameter(description='', name='pattern', parameter_type='string', required=True, default=None),\n",
+              "│   │   │   Parameter(\n",
+              "│   │   │   │   description='',\n",
+              "│   │   │   │   name='excludePatterns',\n",
+              "│   │   │   │   parameter_type='array',\n",
+              "│   │   │   │   required=True,\n",
+              "│   │   │   │   default=None\n",
+              "│   │   │   )\n",
+              "│   │   ],\n",
+              "│   │   provider_id='model-context-protocol',\n",
+              "│   │   provider_resource_id='search_files',\n",
+              "│   │   tool_host='model_context_protocol',\n",
+              "│   │   toolgroup_id='mcp::filesystem',\n",
+              "│   │   type='tool',\n",
+              "│   │   metadata={'endpoint': 'http://localhost:8000/sse'}\n",
+              "│   ),\n",
+              "│   Tool(\n",
+              "│   │   description='Retrieve detailed metadata about a file or directory. Returns comprehensive information including size, creation time, last modified time, permissions, and type. This tool is perfect for understanding file characteristics without reading the actual content. Only works within allowed directories.',\n",
+              "│   │   identifier='get_file_info',\n",
+              "│   │   parameters=[Parameter(description='', name='path', parameter_type='string', required=True, default=None)],\n",
+              "│   │   provider_id='model-context-protocol',\n",
+              "│   │   provider_resource_id='get_file_info',\n",
+              "│   │   tool_host='model_context_protocol',\n",
+              "│   │   toolgroup_id='mcp::filesystem',\n",
+              "│   │   type='tool',\n",
+              "│   │   metadata={'endpoint': 'http://localhost:8000/sse'}\n",
+              "│   ),\n",
+              "│   Tool(\n",
+              "│   │   description='Returns the list of directories that this server is allowed to access. Use this to understand which directories are available before trying to access files.',\n",
+              "│   │   identifier='list_allowed_directories',\n",
+              "│   │   parameters=[],\n",
+              "│   │   provider_id='model-context-protocol',\n",
+              "│   │   provider_resource_id='list_allowed_directories',\n",
+              "│   │   tool_host='model_context_protocol',\n",
+              "│   │   toolgroup_id='mcp::filesystem',\n",
+              "│   │   type='tool',\n",
+              "│   │   metadata={'endpoint': 'http://localhost:8000/sse'}\n",
+              "│   )\n",
+              "]\n",
+              "
\n" + ], + "text/plain": [ + "\u001b[1m[\u001b[0m\n", + "\u001b[2;32m│ \u001b[0m\u001b[1;35mTool\u001b[0m\u001b[1m(\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mdescription\u001b[0m=\u001b[32m'Read the complete contents of a file from the file system. Handles various text encodings and provides detailed error messages if the file cannot be read. Use this tool when you need to examine the contents of a single file. Only works within allowed directories.'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33midentifier\u001b[0m=\u001b[32m'read_file'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mparameters\u001b[0m=\u001b[1m[\u001b[0m\u001b[1;35mParameter\u001b[0m\u001b[1m(\u001b[0m\u001b[33mdescription\u001b[0m=\u001b[32m''\u001b[0m, \u001b[33mname\u001b[0m=\u001b[32m'path'\u001b[0m, \u001b[33mparameter_type\u001b[0m=\u001b[32m'string'\u001b[0m, \u001b[33mrequired\u001b[0m=\u001b[3;92mTrue\u001b[0m, \u001b[33mdefault\u001b[0m=\u001b[3;35mNone\u001b[0m\u001b[1m)\u001b[0m\u001b[1m]\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mprovider_id\u001b[0m=\u001b[32m'model-context-protocol'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mprovider_resource_id\u001b[0m=\u001b[32m'read_file'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mtool_host\u001b[0m=\u001b[32m'model_context_protocol'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mtoolgroup_id\u001b[0m=\u001b[32m'mcp::filesystem'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mtype\u001b[0m=\u001b[32m'tool'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'endpoint'\u001b[0m: \u001b[32m'http://localhost:8000/sse'\u001b[0m\u001b[1m}\u001b[0m\n", + "\u001b[2;32m│ \u001b[0m\u001b[1m)\u001b[0m,\n", + "\u001b[2;32m│ \u001b[0m\u001b[1;35mTool\u001b[0m\u001b[1m(\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mdescription\u001b[0m=\u001b[32m\"Read\u001b[0m\u001b[32m the contents of multiple files simultaneously. This is more efficient than reading files one by one when you need to analyze or compare multiple files. Each file's content is returned with its path as a reference. Failed reads for individual files won't stop the entire operation. Only works within allowed directories.\"\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33midentifier\u001b[0m=\u001b[32m'read_multiple_files'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mparameters\u001b[0m=\u001b[1m[\u001b[0m\u001b[1;35mParameter\u001b[0m\u001b[1m(\u001b[0m\u001b[33mdescription\u001b[0m=\u001b[32m''\u001b[0m, \u001b[33mname\u001b[0m=\u001b[32m'paths'\u001b[0m, \u001b[33mparameter_type\u001b[0m=\u001b[32m'array'\u001b[0m, \u001b[33mrequired\u001b[0m=\u001b[3;92mTrue\u001b[0m, \u001b[33mdefault\u001b[0m=\u001b[3;35mNone\u001b[0m\u001b[1m)\u001b[0m\u001b[1m]\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mprovider_id\u001b[0m=\u001b[32m'model-context-protocol'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mprovider_resource_id\u001b[0m=\u001b[32m'read_multiple_files'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mtool_host\u001b[0m=\u001b[32m'model_context_protocol'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mtoolgroup_id\u001b[0m=\u001b[32m'mcp::filesystem'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mtype\u001b[0m=\u001b[32m'tool'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'endpoint'\u001b[0m: \u001b[32m'http://localhost:8000/sse'\u001b[0m\u001b[1m}\u001b[0m\n", + "\u001b[2;32m│ \u001b[0m\u001b[1m)\u001b[0m,\n", + "\u001b[2;32m│ \u001b[0m\u001b[1;35mTool\u001b[0m\u001b[1m(\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mdescription\u001b[0m=\u001b[32m'Create a new file or completely overwrite an existing file with new content. Use with caution as it will overwrite existing files without warning. Handles text content with proper encoding. Only works within allowed directories.'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33midentifier\u001b[0m=\u001b[32m'write_file'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mparameters\u001b[0m=\u001b[1m[\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[1;35mParameter\u001b[0m\u001b[1m(\u001b[0m\u001b[33mdescription\u001b[0m=\u001b[32m''\u001b[0m, \u001b[33mname\u001b[0m=\u001b[32m'path'\u001b[0m, \u001b[33mparameter_type\u001b[0m=\u001b[32m'string'\u001b[0m, \u001b[33mrequired\u001b[0m=\u001b[3;92mTrue\u001b[0m, \u001b[33mdefault\u001b[0m=\u001b[3;35mNone\u001b[0m\u001b[1m)\u001b[0m,\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[1;35mParameter\u001b[0m\u001b[1m(\u001b[0m\u001b[33mdescription\u001b[0m=\u001b[32m''\u001b[0m, \u001b[33mname\u001b[0m=\u001b[32m'content'\u001b[0m, \u001b[33mparameter_type\u001b[0m=\u001b[32m'string'\u001b[0m, \u001b[33mrequired\u001b[0m=\u001b[3;92mTrue\u001b[0m, \u001b[33mdefault\u001b[0m=\u001b[3;35mNone\u001b[0m\u001b[1m)\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[1m]\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mprovider_id\u001b[0m=\u001b[32m'model-context-protocol'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mprovider_resource_id\u001b[0m=\u001b[32m'write_file'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mtool_host\u001b[0m=\u001b[32m'model_context_protocol'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mtoolgroup_id\u001b[0m=\u001b[32m'mcp::filesystem'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mtype\u001b[0m=\u001b[32m'tool'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'endpoint'\u001b[0m: \u001b[32m'http://localhost:8000/sse'\u001b[0m\u001b[1m}\u001b[0m\n", + "\u001b[2;32m│ \u001b[0m\u001b[1m)\u001b[0m,\n", + "\u001b[2;32m│ \u001b[0m\u001b[1;35mTool\u001b[0m\u001b[1m(\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mdescription\u001b[0m=\u001b[32m'Make line-based edits to a text file. Each edit replaces exact line sequences with new content. Returns a git-style diff showing the changes made. Only works within allowed directories.'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33midentifier\u001b[0m=\u001b[32m'edit_file'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mparameters\u001b[0m=\u001b[1m[\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[1;35mParameter\u001b[0m\u001b[1m(\u001b[0m\u001b[33mdescription\u001b[0m=\u001b[32m''\u001b[0m, \u001b[33mname\u001b[0m=\u001b[32m'path'\u001b[0m, \u001b[33mparameter_type\u001b[0m=\u001b[32m'string'\u001b[0m, \u001b[33mrequired\u001b[0m=\u001b[3;92mTrue\u001b[0m, \u001b[33mdefault\u001b[0m=\u001b[3;35mNone\u001b[0m\u001b[1m)\u001b[0m,\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[1;35mParameter\u001b[0m\u001b[1m(\u001b[0m\u001b[33mdescription\u001b[0m=\u001b[32m''\u001b[0m, \u001b[33mname\u001b[0m=\u001b[32m'edits'\u001b[0m, \u001b[33mparameter_type\u001b[0m=\u001b[32m'array'\u001b[0m, \u001b[33mrequired\u001b[0m=\u001b[3;92mTrue\u001b[0m, \u001b[33mdefault\u001b[0m=\u001b[3;35mNone\u001b[0m\u001b[1m)\u001b[0m,\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[1;35mParameter\u001b[0m\u001b[1m(\u001b[0m\n", + "\u001b[2;32m│ │ │ │ \u001b[0m\u001b[33mdescription\u001b[0m=\u001b[32m'Preview changes using git-style diff format'\u001b[0m,\n", + "\u001b[2;32m│ │ │ │ \u001b[0m\u001b[33mname\u001b[0m=\u001b[32m'dryRun'\u001b[0m,\n", + "\u001b[2;32m│ │ │ │ \u001b[0m\u001b[33mparameter_type\u001b[0m=\u001b[32m'boolean'\u001b[0m,\n", + "\u001b[2;32m│ │ │ │ \u001b[0m\u001b[33mrequired\u001b[0m=\u001b[3;92mTrue\u001b[0m,\n", + "\u001b[2;32m│ │ │ │ \u001b[0m\u001b[33mdefault\u001b[0m=\u001b[3;35mNone\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[1m)\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[1m]\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mprovider_id\u001b[0m=\u001b[32m'model-context-protocol'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mprovider_resource_id\u001b[0m=\u001b[32m'edit_file'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mtool_host\u001b[0m=\u001b[32m'model_context_protocol'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mtoolgroup_id\u001b[0m=\u001b[32m'mcp::filesystem'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mtype\u001b[0m=\u001b[32m'tool'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'endpoint'\u001b[0m: \u001b[32m'http://localhost:8000/sse'\u001b[0m\u001b[1m}\u001b[0m\n", + "\u001b[2;32m│ \u001b[0m\u001b[1m)\u001b[0m,\n", + "\u001b[2;32m│ \u001b[0m\u001b[1;35mTool\u001b[0m\u001b[1m(\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mdescription\u001b[0m=\u001b[32m'Create a new directory or ensure a directory exists. Can create multiple nested directories in one operation. If the directory already exists, this operation will succeed silently. Perfect for setting up directory structures for projects or ensuring required paths exist. Only works within allowed directories.'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33midentifier\u001b[0m=\u001b[32m'create_directory'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mparameters\u001b[0m=\u001b[1m[\u001b[0m\u001b[1;35mParameter\u001b[0m\u001b[1m(\u001b[0m\u001b[33mdescription\u001b[0m=\u001b[32m''\u001b[0m, \u001b[33mname\u001b[0m=\u001b[32m'path'\u001b[0m, \u001b[33mparameter_type\u001b[0m=\u001b[32m'string'\u001b[0m, \u001b[33mrequired\u001b[0m=\u001b[3;92mTrue\u001b[0m, \u001b[33mdefault\u001b[0m=\u001b[3;35mNone\u001b[0m\u001b[1m)\u001b[0m\u001b[1m]\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mprovider_id\u001b[0m=\u001b[32m'model-context-protocol'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mprovider_resource_id\u001b[0m=\u001b[32m'create_directory'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mtool_host\u001b[0m=\u001b[32m'model_context_protocol'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mtoolgroup_id\u001b[0m=\u001b[32m'mcp::filesystem'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mtype\u001b[0m=\u001b[32m'tool'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'endpoint'\u001b[0m: \u001b[32m'http://localhost:8000/sse'\u001b[0m\u001b[1m}\u001b[0m\n", + "\u001b[2;32m│ \u001b[0m\u001b[1m)\u001b[0m,\n", + "\u001b[2;32m│ \u001b[0m\u001b[1;35mTool\u001b[0m\u001b[1m(\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mdescription\u001b[0m=\u001b[32m'Get a detailed listing of all files and directories in a specified path. Results clearly distinguish between files and directories with \u001b[0m\u001b[32m[\u001b[0m\u001b[32mFILE\u001b[0m\u001b[32m]\u001b[0m\u001b[32m and \u001b[0m\u001b[32m[\u001b[0m\u001b[32mDIR\u001b[0m\u001b[32m]\u001b[0m\u001b[32m prefixes. This tool is essential for understanding directory structure and finding specific files within a directory. Only works within allowed directories.'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33midentifier\u001b[0m=\u001b[32m'list_directory'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mparameters\u001b[0m=\u001b[1m[\u001b[0m\u001b[1;35mParameter\u001b[0m\u001b[1m(\u001b[0m\u001b[33mdescription\u001b[0m=\u001b[32m''\u001b[0m, \u001b[33mname\u001b[0m=\u001b[32m'path'\u001b[0m, \u001b[33mparameter_type\u001b[0m=\u001b[32m'string'\u001b[0m, \u001b[33mrequired\u001b[0m=\u001b[3;92mTrue\u001b[0m, \u001b[33mdefault\u001b[0m=\u001b[3;35mNone\u001b[0m\u001b[1m)\u001b[0m\u001b[1m]\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mprovider_id\u001b[0m=\u001b[32m'model-context-protocol'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mprovider_resource_id\u001b[0m=\u001b[32m'list_directory'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mtool_host\u001b[0m=\u001b[32m'model_context_protocol'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mtoolgroup_id\u001b[0m=\u001b[32m'mcp::filesystem'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mtype\u001b[0m=\u001b[32m'tool'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'endpoint'\u001b[0m: \u001b[32m'http://localhost:8000/sse'\u001b[0m\u001b[1m}\u001b[0m\n", + "\u001b[2;32m│ \u001b[0m\u001b[1m)\u001b[0m,\n", + "\u001b[2;32m│ \u001b[0m\u001b[1;35mTool\u001b[0m\u001b[1m(\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mdescription\u001b[0m=\u001b[32m\"Get\u001b[0m\u001b[32m a recursive tree view of files and directories as a JSON structure. Each entry includes 'name', 'type' \u001b[0m\u001b[32m(\u001b[0m\u001b[32mfile/directory\u001b[0m\u001b[32m)\u001b[0m\u001b[32m, and 'children' for directories. Files have no children array, while directories always have a children array \u001b[0m\u001b[32m(\u001b[0m\u001b[32mwhich may be empty\u001b[0m\u001b[32m)\u001b[0m\u001b[32m. The output is formatted with 2-space indentation for readability. Only works within allowed directories.\"\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33midentifier\u001b[0m=\u001b[32m'directory_tree'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mparameters\u001b[0m=\u001b[1m[\u001b[0m\u001b[1;35mParameter\u001b[0m\u001b[1m(\u001b[0m\u001b[33mdescription\u001b[0m=\u001b[32m''\u001b[0m, \u001b[33mname\u001b[0m=\u001b[32m'path'\u001b[0m, \u001b[33mparameter_type\u001b[0m=\u001b[32m'string'\u001b[0m, \u001b[33mrequired\u001b[0m=\u001b[3;92mTrue\u001b[0m, \u001b[33mdefault\u001b[0m=\u001b[3;35mNone\u001b[0m\u001b[1m)\u001b[0m\u001b[1m]\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mprovider_id\u001b[0m=\u001b[32m'model-context-protocol'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mprovider_resource_id\u001b[0m=\u001b[32m'directory_tree'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mtool_host\u001b[0m=\u001b[32m'model_context_protocol'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mtoolgroup_id\u001b[0m=\u001b[32m'mcp::filesystem'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mtype\u001b[0m=\u001b[32m'tool'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'endpoint'\u001b[0m: \u001b[32m'http://localhost:8000/sse'\u001b[0m\u001b[1m}\u001b[0m\n", + "\u001b[2;32m│ \u001b[0m\u001b[1m)\u001b[0m,\n", + "\u001b[2;32m│ \u001b[0m\u001b[1;35mTool\u001b[0m\u001b[1m(\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mdescription\u001b[0m=\u001b[32m'Move or rename files and directories. Can move files between directories and rename them in a single operation. If the destination exists, the operation will fail. Works across different directories and can be used for simple renaming within the same directory. Both source and destination must be within allowed directories.'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33midentifier\u001b[0m=\u001b[32m'move_file'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mparameters\u001b[0m=\u001b[1m[\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[1;35mParameter\u001b[0m\u001b[1m(\u001b[0m\u001b[33mdescription\u001b[0m=\u001b[32m''\u001b[0m, \u001b[33mname\u001b[0m=\u001b[32m'source'\u001b[0m, \u001b[33mparameter_type\u001b[0m=\u001b[32m'string'\u001b[0m, \u001b[33mrequired\u001b[0m=\u001b[3;92mTrue\u001b[0m, \u001b[33mdefault\u001b[0m=\u001b[3;35mNone\u001b[0m\u001b[1m)\u001b[0m,\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[1;35mParameter\u001b[0m\u001b[1m(\u001b[0m\u001b[33mdescription\u001b[0m=\u001b[32m''\u001b[0m, \u001b[33mname\u001b[0m=\u001b[32m'destination'\u001b[0m, \u001b[33mparameter_type\u001b[0m=\u001b[32m'string'\u001b[0m, \u001b[33mrequired\u001b[0m=\u001b[3;92mTrue\u001b[0m, \u001b[33mdefault\u001b[0m=\u001b[3;35mNone\u001b[0m\u001b[1m)\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[1m]\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mprovider_id\u001b[0m=\u001b[32m'model-context-protocol'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mprovider_resource_id\u001b[0m=\u001b[32m'move_file'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mtool_host\u001b[0m=\u001b[32m'model_context_protocol'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mtoolgroup_id\u001b[0m=\u001b[32m'mcp::filesystem'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mtype\u001b[0m=\u001b[32m'tool'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'endpoint'\u001b[0m: \u001b[32m'http://localhost:8000/sse'\u001b[0m\u001b[1m}\u001b[0m\n", + "\u001b[2;32m│ \u001b[0m\u001b[1m)\u001b[0m,\n", + "\u001b[2;32m│ \u001b[0m\u001b[1;35mTool\u001b[0m\u001b[1m(\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mdescription\u001b[0m=\u001b[32m\"Recursively\u001b[0m\u001b[32m search for files and directories matching a pattern. Searches through all subdirectories from the starting path. The search is case-insensitive and matches partial names. Returns full paths to all matching items. Great for finding files when you don't know their exact location. Only searches within allowed directories.\"\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33midentifier\u001b[0m=\u001b[32m'search_files'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mparameters\u001b[0m=\u001b[1m[\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[1;35mParameter\u001b[0m\u001b[1m(\u001b[0m\u001b[33mdescription\u001b[0m=\u001b[32m''\u001b[0m, \u001b[33mname\u001b[0m=\u001b[32m'path'\u001b[0m, \u001b[33mparameter_type\u001b[0m=\u001b[32m'string'\u001b[0m, \u001b[33mrequired\u001b[0m=\u001b[3;92mTrue\u001b[0m, \u001b[33mdefault\u001b[0m=\u001b[3;35mNone\u001b[0m\u001b[1m)\u001b[0m,\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[1;35mParameter\u001b[0m\u001b[1m(\u001b[0m\u001b[33mdescription\u001b[0m=\u001b[32m''\u001b[0m, \u001b[33mname\u001b[0m=\u001b[32m'pattern'\u001b[0m, \u001b[33mparameter_type\u001b[0m=\u001b[32m'string'\u001b[0m, \u001b[33mrequired\u001b[0m=\u001b[3;92mTrue\u001b[0m, \u001b[33mdefault\u001b[0m=\u001b[3;35mNone\u001b[0m\u001b[1m)\u001b[0m,\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[1;35mParameter\u001b[0m\u001b[1m(\u001b[0m\n", + "\u001b[2;32m│ │ │ │ \u001b[0m\u001b[33mdescription\u001b[0m=\u001b[32m''\u001b[0m,\n", + "\u001b[2;32m│ │ │ │ \u001b[0m\u001b[33mname\u001b[0m=\u001b[32m'excludePatterns'\u001b[0m,\n", + "\u001b[2;32m│ │ │ │ \u001b[0m\u001b[33mparameter_type\u001b[0m=\u001b[32m'array'\u001b[0m,\n", + "\u001b[2;32m│ │ │ │ \u001b[0m\u001b[33mrequired\u001b[0m=\u001b[3;92mTrue\u001b[0m,\n", + "\u001b[2;32m│ │ │ │ \u001b[0m\u001b[33mdefault\u001b[0m=\u001b[3;35mNone\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[1m)\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[1m]\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mprovider_id\u001b[0m=\u001b[32m'model-context-protocol'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mprovider_resource_id\u001b[0m=\u001b[32m'search_files'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mtool_host\u001b[0m=\u001b[32m'model_context_protocol'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mtoolgroup_id\u001b[0m=\u001b[32m'mcp::filesystem'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mtype\u001b[0m=\u001b[32m'tool'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'endpoint'\u001b[0m: \u001b[32m'http://localhost:8000/sse'\u001b[0m\u001b[1m}\u001b[0m\n", + "\u001b[2;32m│ \u001b[0m\u001b[1m)\u001b[0m,\n", + "\u001b[2;32m│ \u001b[0m\u001b[1;35mTool\u001b[0m\u001b[1m(\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mdescription\u001b[0m=\u001b[32m'Retrieve detailed metadata about a file or directory. Returns comprehensive information including size, creation time, last modified time, permissions, and type. This tool is perfect for understanding file characteristics without reading the actual content. Only works within allowed directories.'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33midentifier\u001b[0m=\u001b[32m'get_file_info'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mparameters\u001b[0m=\u001b[1m[\u001b[0m\u001b[1;35mParameter\u001b[0m\u001b[1m(\u001b[0m\u001b[33mdescription\u001b[0m=\u001b[32m''\u001b[0m, \u001b[33mname\u001b[0m=\u001b[32m'path'\u001b[0m, \u001b[33mparameter_type\u001b[0m=\u001b[32m'string'\u001b[0m, \u001b[33mrequired\u001b[0m=\u001b[3;92mTrue\u001b[0m, \u001b[33mdefault\u001b[0m=\u001b[3;35mNone\u001b[0m\u001b[1m)\u001b[0m\u001b[1m]\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mprovider_id\u001b[0m=\u001b[32m'model-context-protocol'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mprovider_resource_id\u001b[0m=\u001b[32m'get_file_info'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mtool_host\u001b[0m=\u001b[32m'model_context_protocol'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mtoolgroup_id\u001b[0m=\u001b[32m'mcp::filesystem'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mtype\u001b[0m=\u001b[32m'tool'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'endpoint'\u001b[0m: \u001b[32m'http://localhost:8000/sse'\u001b[0m\u001b[1m}\u001b[0m\n", + "\u001b[2;32m│ \u001b[0m\u001b[1m)\u001b[0m,\n", + "\u001b[2;32m│ \u001b[0m\u001b[1;35mTool\u001b[0m\u001b[1m(\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mdescription\u001b[0m=\u001b[32m'Returns the list of directories that this server is allowed to access. Use this to understand which directories are available before trying to access files.'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33midentifier\u001b[0m=\u001b[32m'list_allowed_directories'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mparameters\u001b[0m=\u001b[1m[\u001b[0m\u001b[1m]\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mprovider_id\u001b[0m=\u001b[32m'model-context-protocol'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mprovider_resource_id\u001b[0m=\u001b[32m'list_allowed_directories'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mtool_host\u001b[0m=\u001b[32m'model_context_protocol'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mtoolgroup_id\u001b[0m=\u001b[32m'mcp::filesystem'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mtype\u001b[0m=\u001b[32m'tool'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'endpoint'\u001b[0m: \u001b[32m'http://localhost:8000/sse'\u001b[0m\u001b[1m}\u001b[0m\n", + "\u001b[2;32m│ \u001b[0m\u001b[1m)\u001b[0m\n", + "\u001b[1m]\u001b[0m\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "pprint(client.tools.list(toolgroup_id=\"mcp::filesystem\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "vttLbj_YO01f", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "vttLbj_YO01f", + "outputId": "04bc486c-3a61-49c6-d0d2-4a211d6de0b5" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "User> Hello\n", + "inference> None of the provided functions can be used to respond to a greeting.\n", + "User> list all the files /content\n", + "inference> {\"type\": \"function\", \"name\": \"list_directory\", \"parameters\": {\"path\": \"/content\"}}\n", + "tool_execution> Tool:list_directory Args:{'path': '/content'}\n", + "tool_execution> Tool:list_directory Response:{\"type\":\"text\",\"text\":\"[DIR] .config\\n[FILE] bar\\n[FILE] foo\\n[DIR] sample_data\"}\n", + "inference> {\"type\": \"function\", \"name\": \"list_directory\", \"parameters\": {\"path\": \"/content\"}}\n", + "tool_execution> Tool:list_directory Args:{'path': '/content'}\n", + "tool_execution> Tool:list_directory Response:{\"type\":\"text\",\"text\":\"[DIR] .config\\n[FILE] bar\\n[FILE] foo\\n[DIR] sample_data\"}\n", + "inference> The list of files in the /content directory is:\n", + "\n", + "[DIR] .config\n", + "[FILE] bar\n", + "[FILE] foo\n", + "[DIR] sample_data\n" + ] + } + ], + "source": [ + "from llama_stack_client.lib.agents.agent import Agent\n", + "from llama_stack_client.lib.agents.event_logger import EventLogger\n", + "from llama_stack_client.types.agent_create_params import AgentConfig\n", + "from termcolor import cprint\n", + "\n", + "agent_config = AgentConfig(\n", + " model=model_id,\n", + " instructions=\"You are a helpful assistant\",\n", + " toolgroups=[\"mcp::filesystem\"],\n", + " input_shields=[],\n", + " output_shields=[],\n", + " enable_session_persistence=False,\n", + ")\n", + "agent = Agent(client, agent_config)\n", + "user_prompts = [\n", + " \"Hello\",\n", + " \"list all the files /content\",\n", + "]\n", + "\n", + "session_id = agent.create_session(\"test-session\")\n", + "for prompt in user_prompts:\n", + " cprint(f\"User> {prompt}\", \"green\")\n", + " response = agent.create_turn(\n", + " messages=[\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": prompt,\n", + " }\n", + " ],\n", + " session_id=session_id,\n", + " )\n", + " for log in EventLogger().log(response):\n", + " log.print()\n" + ] + }, { "cell_type": "markdown", "id": "FJ85DUhgBZd7", @@ -1887,164 +3228,6 @@ "- In this example, we will show how to build an Agent with Llama Stack, and query the agent's traces into an online dataset that can be used for evaluation. " ] }, - { - "cell_type": "markdown", - "id": "_JueJAKyJR5m", - "metadata": { - "id": "_JueJAKyJR5m" - }, - "source": [ - "##### 🚧 Patches 🚧\n", - "- The following cells are temporary patches to get `telemetry` working." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "klPkK1t7CzIY", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "collapsed": true, - "id": "klPkK1t7CzIY", - "outputId": "ab0c1490-7fa6-446c-8e35-7b42f57e8a04" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found existing installation: llama_stack 0.0.61\n", - "Uninstalling llama_stack-0.0.61:\n", - " Would remove:\n", - " /usr/local/bin/install-wheel-from-presigned\n", - " /usr/local/bin/llama\n", - " /usr/local/lib/python3.10/dist-packages/llama_stack-0.0.61.dist-info/*\n", - " /usr/local/lib/python3.10/dist-packages/llama_stack/*\n", - "Proceed (Y/n)? Y\n", - " Successfully uninstalled llama_stack-0.0.61\n", - "Collecting git+https://github.com/meta-llama/llama-stack.git@main\n", - " Cloning https://github.com/meta-llama/llama-stack.git (to revision main) to /tmp/pip-req-build-oryyzdm1\n", - " Running command git clone --filter=blob:none --quiet https://github.com/meta-llama/llama-stack.git /tmp/pip-req-build-oryyzdm1\n", - " Resolved https://github.com/meta-llama/llama-stack.git to commit 53b3a1e345c46d7d37c1af3d675092a4cbfe85f9\n", - " Running command git submodule update --init --recursive -q\n", - " Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", - " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", - " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", - "Requirement already satisfied: blobfile in /usr/local/lib/python3.10/dist-packages (from llama_stack==0.0.61) (3.0.0)\n", - "Requirement already satisfied: fire in /usr/local/lib/python3.10/dist-packages (from llama_stack==0.0.61) (0.7.0)\n", - "Requirement already satisfied: httpx in /usr/local/lib/python3.10/dist-packages (from llama_stack==0.0.61) (0.28.1)\n", - "Requirement already satisfied: huggingface-hub in /usr/local/lib/python3.10/dist-packages (from llama_stack==0.0.61) (0.26.5)\n", - "Requirement already satisfied: llama-models>=0.0.61 in /usr/local/lib/python3.10/dist-packages (from llama_stack==0.0.61) (0.0.61)\n", - "Requirement already satisfied: llama-stack-client>=0.0.61 in /usr/local/lib/python3.10/dist-packages (from llama_stack==0.0.61) (0.0.61)\n", - "Requirement already satisfied: prompt-toolkit in /usr/local/lib/python3.10/dist-packages (from llama_stack==0.0.61) (3.0.48)\n", - "Requirement already satisfied: python-dotenv in /usr/local/lib/python3.10/dist-packages (from llama_stack==0.0.61) (1.0.1)\n", - "Requirement already satisfied: pydantic>=2 in /usr/local/lib/python3.10/dist-packages (from llama_stack==0.0.61) (2.10.3)\n", - "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from llama_stack==0.0.61) (2.32.3)\n", - "Requirement already satisfied: rich in /usr/local/lib/python3.10/dist-packages (from llama_stack==0.0.61) (13.9.4)\n", - "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from llama_stack==0.0.61) (75.1.0)\n", - "Requirement already satisfied: termcolor in /usr/local/lib/python3.10/dist-packages (from llama_stack==0.0.61) (2.5.0)\n", - "Requirement already satisfied: PyYAML in /usr/local/lib/python3.10/dist-packages (from llama-models>=0.0.61->llama_stack==0.0.61) (6.0.2)\n", - "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from llama-models>=0.0.61->llama_stack==0.0.61) (3.1.4)\n", - "Requirement already satisfied: tiktoken in /usr/local/lib/python3.10/dist-packages (from llama-models>=0.0.61->llama_stack==0.0.61) (0.8.0)\n", - "Requirement already satisfied: Pillow in /usr/local/lib/python3.10/dist-packages (from llama-models>=0.0.61->llama_stack==0.0.61) (10.4.0)\n", - "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.61->llama_stack==0.0.61) (3.7.1)\n", - "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.61->llama_stack==0.0.61) (8.1.7)\n", - "Requirement already satisfied: distro<2,>=1.7.0 in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.61->llama_stack==0.0.61) (1.9.0)\n", - "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.61->llama_stack==0.0.61) (2.2.2)\n", - "Requirement already satisfied: pyaml in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.61->llama_stack==0.0.61) (24.12.1)\n", - "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.61->llama_stack==0.0.61) (1.3.1)\n", - "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.61->llama_stack==0.0.61) (4.66.6)\n", - "Requirement already satisfied: typing-extensions<5,>=4.7 in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.61->llama_stack==0.0.61) (4.12.2)\n", - "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx->llama_stack==0.0.61) (2024.8.30)\n", - "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx->llama_stack==0.0.61) (1.0.7)\n", - "Requirement already satisfied: idna in /usr/local/lib/python3.10/dist-packages (from httpx->llama_stack==0.0.61) (3.10)\n", - "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx->llama_stack==0.0.61) (0.14.0)\n", - "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.10/dist-packages (from pydantic>=2->llama_stack==0.0.61) (0.7.0)\n", - "Requirement already satisfied: pydantic-core==2.27.1 in /usr/local/lib/python3.10/dist-packages (from pydantic>=2->llama_stack==0.0.61) (2.27.1)\n", - "Requirement already satisfied: pycryptodomex>=3.8 in /usr/local/lib/python3.10/dist-packages (from blobfile->llama_stack==0.0.61) (3.21.0)\n", - "Requirement already satisfied: urllib3<3,>=1.25.3 in /usr/local/lib/python3.10/dist-packages (from blobfile->llama_stack==0.0.61) (2.2.3)\n", - "Requirement already satisfied: lxml>=4.9 in /usr/local/lib/python3.10/dist-packages (from blobfile->llama_stack==0.0.61) (5.3.0)\n", - "Requirement already satisfied: filelock>=3.0 in /usr/local/lib/python3.10/dist-packages (from blobfile->llama_stack==0.0.61) (3.16.1)\n", - "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->llama_stack==0.0.61) (2024.9.0)\n", - "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->llama_stack==0.0.61) (24.2)\n", - "Requirement already satisfied: wcwidth in /usr/local/lib/python3.10/dist-packages (from prompt-toolkit->llama_stack==0.0.61) (0.2.13)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->llama_stack==0.0.61) (3.4.0)\n", - "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich->llama_stack==0.0.61) (3.0.0)\n", - "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich->llama_stack==0.0.61) (2.18.0)\n", - "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->llama-stack-client>=0.0.61->llama_stack==0.0.61) (1.2.2)\n", - "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich->llama_stack==0.0.61) (0.1.2)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->llama-models>=0.0.61->llama_stack==0.0.61) (3.0.2)\n", - "Requirement already satisfied: numpy>=1.22.4 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-stack-client>=0.0.61->llama_stack==0.0.61) (1.26.4)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-stack-client>=0.0.61->llama_stack==0.0.61) (2.8.2)\n", - "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-stack-client>=0.0.61->llama_stack==0.0.61) (2024.2)\n", - "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-stack-client>=0.0.61->llama_stack==0.0.61) (2024.2)\n", - "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken->llama-models>=0.0.61->llama_stack==0.0.61) (2024.9.11)\n", - "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas->llama-stack-client>=0.0.61->llama_stack==0.0.61) (1.17.0)\n", - "Building wheels for collected packages: llama_stack\n", - " Building wheel for llama_stack (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", - " Created wheel for llama_stack: filename=llama_stack-0.0.61-py3-none-any.whl size=464145 sha256=da71747aceef9aec43553f66c43095486d1a920e47bb0e47e2729a8e4328fff6\n", - " Stored in directory: /tmp/pip-ephem-wheel-cache-jquw5j7f/wheels/74/e4/3b/079983408fa9323c1f2807e404ee78b468c74bec381eb70d4f\n", - "Successfully built llama_stack\n", - "Installing collected packages: llama_stack\n", - "Successfully installed llama_stack-0.0.61\n" - ] - }, - { - "data": { - "application/vnd.colab-display-data+json": { - "id": "7701cb0c982f4250a46721fededf9647", - "pip_warning": { - "packages": [ - "llama_stack" - ] - } - } - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# need to install on latest main\n", - "!pip uninstall llama-stack\n", - "!pip install git+https://github.com/meta-llama/llama-stack.git@main" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9jJ75JlnETTH", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "9jJ75JlnETTH", - "outputId": "76bd3912-f814-428c-88e1-c1113af77856" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Removed handler StreamHandler from root logger\n" - ] - } - ], - "source": [ - "# disable logging for clean server logs\n", - "import logging\n", - "def remove_root_handlers():\n", - " root_logger = logging.getLogger()\n", - " for handler in root_logger.handlers[:]:\n", - " root_logger.removeHandler(handler)\n", - " print(f\"Removed handler {handler.__class__.__name__} from root logger\")\n", - "\n", - "\n", - "remove_root_handlers()" - ] - }, { "cell_type": "markdown", "id": "_t_tcWq0JcJ4", @@ -2057,49 +3240,46 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "id": "4iCO59kP20Zs", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "4iCO59kP20Zs", - "outputId": "f6179de6-054d-4452-a893-8d9b64c5a0d1" + "outputId": "894c6333-30e9-4f1e-9b63-1bfb1cae51e2" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "inference> Let me check the latest sports news.\n", - "inference> bravy_search.call(query=\"Bill Cosby South Park episode\")\n", - "CustomTool> Unknown tool `bravy_search` was called.\n", - "inference> brave_search.call(query=\"Andrew Tate kickboxing name\")\n", - "tool_execution> Tool:brave_search Args:{'query': 'Andrew Tate kickboxing name'}\n", - "tool_execution> Tool:brave_search Response:{\"query\": \"Andrew Tate kickboxing name\", \"top_k\": [{\"title\": \"Andrew Tate kickboxing record: How many championships ... - FirstSportz\", \"url\": \"https://firstsportz.com/mma-how-many-championships-does-andrew-tate-have/\", \"content\": \"Andrew Tate's Kickboxing career. During his kickboxing career, he used the nickname \\\"King Cobra,\\\" which he currently uses as his Twitter name. Tate had an unorthodox style of movement inside the ring. He kept his hands down most of the time and relied on quick jabs and an overhand right to land significant strikes.\", \"score\": 0.9996244, \"raw_content\": null}, {\"title\": \"Andrew Tate: Kickboxing Record, Facts, Height, Weight, Age, Biography\", \"url\": \"https://www.lowkickmma.com/andrew-tate-kickboxing-record-facts-height-weight-age-biography/\", \"content\": \"Birth Name: Emory Andrew Tate III: Date of Birth: 1 December 1986: Place of Birth: Washington, D.C., U.S. ... In his professional kickboxing career, Andrew Tate won 32 of his fights by knockout.\", \"score\": 0.99909246, \"raw_content\": null}, {\"title\": \"Who is Andrew Tate? MMA, kickboxing record and controversies of fighter ...\", \"url\": \"https://www.sportingnews.com/us/kickboxing/news/andrew-tate-mma-kickboxing-record-controversies/u50waalc9cfz7krjg9wnyb7p\", \"content\": \"Andrew Tate kickboxing record After launching his career as a 20-year-old in 2007, Tate built a formidable kickboxing record that included 76 wins across 85 fights in more than 13 years in the ring.\", \"score\": 0.9976586, \"raw_content\": null}, {\"title\": \"About Andrew Tate: A Journey from Champion to Controversy\", \"url\": \"https://reachmorpheus.com/andrew-tate/\", \"content\": \"Andrew Tate's kickboxing career, beginning in 2005, is a tale of determination and skill. He quickly made a name for himself in the sport, rising through the ranks with his unique fighting style and strategic approach, honed by his chess-playing background.\", \"score\": 0.99701905, \"raw_content\": null}, {\"title\": \"Andrew Tate Bio, Wiki, Net Worth, Age, Family, MMA Career - Next Biography\", \"url\": \"https://www.nextbiography.com/andrew-tate/\", \"content\": \"Andrew Tate Age. Andrew Tate is 36 years old as of 2023, born on December 1, 1986, in Washington, DC. By his mid-thirties, Andrew Tate has become an esteemed figure in the world of kickboxing, showcasing remarkable expertise and experience in the sport. Early Life of Andrew Tate. Andrew Tate was born on 01 December 1986 to an African-American\", \"score\": 0.99368566, \"raw_content\": null}]}\n", - "shield_call> No Violation\n", - "inference> Andrew Tate's kickboxing name is \"King Cobra.\"\n" + "\u001b[30m\u001b[0m\u001b[33minference> \u001b[0m\u001b[36m\u001b[0m\u001b[36mbr\u001b[0m\u001b[36mave\u001b[0m\u001b[36m_search\u001b[0m\u001b[36m.call\u001b[0m\u001b[36m(query\u001b[0m\u001b[36m=\"\u001b[0m\u001b[36mN\u001b[0m\u001b[36mBA\u001b[0m\u001b[36m Western\u001b[0m\u001b[36m Conference\u001b[0m\u001b[36m Finals\u001b[0m\u001b[36m \u001b[0m\u001b[36m202\u001b[0m\u001b[36m4\u001b[0m\u001b[36m teams\u001b[0m\u001b[36m\")\u001b[0m\u001b[97m\u001b[0m\n", + "\u001b[32mtool_execution> Tool:brave_search Args:{'query': 'NBA Western Conference Finals 2024 teams'}\u001b[0m\n", + "\u001b[32mtool_execution> Tool:brave_search Response:{\"query\": \"NBA Western Conference Finals 2024 teams\", \"top_k\": [{\"title\": \"2024 NBA Western Conference Finals - Basketball-Reference.com\", \"url\": \"https://www.basketball-reference.com/playoffs/2024-nba-western-conference-finals-mavericks-vs-timberwolves.html\", \"content\": \"2024 NBA Western Conference Finals Mavericks vs. Timberwolves League Champion: Boston Celtics. Finals MVP: Jaylen Brown (20.8 / 5.4 / 5.0) 2024 Playoff Leaders: PTS: Luka Don\\u010di\\u0107 (635) TRB: Luka Don\\u010di\\u0107 (208) AST: Luka Don\\u010di\\u0107 (178) WS: Derrick White (2.9) More playoffs info\", \"score\": 0.9310187, \"raw_content\": null}, {\"title\": \"NBA Western Conference Finals 2024: Dates, schedule and more - Sportskeeda\", \"url\": \"https://www.sportskeeda.com/basketball/news-nba-western-conference-finals-2024-dates-schedule-and-more\", \"content\": \"NBA Western Conference Finals 2024: Dates & Schedule The 2023-24 NBA Western Conference Finals will start on Wednesday, May 22. The Mavericks will face the team that wins in Game 7 between the\", \"score\": 0.8914433, \"raw_content\": null}, {\"title\": \"2024 Playoffs: West Finals | Timberwolves (3) vs. Mavericks (5) - NBA.com\", \"url\": \"https://www.nba.com/playoffs/2024/west-final\", \"content\": \"The Dallas Mavericks and Minnesota Timberwolves have advanced to the 2024 Western Conference Finals during the NBA playoffs.\", \"score\": 0.8884594, \"raw_content\": null}, {\"title\": \"2024 NBA Western Conference playoff bracket - Basketnews.com\", \"url\": \"https://basketnews.com/news-204687-2024-nba-western-conference-playoff-bracket.html\", \"content\": \"In the 2024 NBA Western Conference playoffs, the Oklahoma City Thunder clinched the No. 1 seed. Every team from the Western Conference played their final game of the regular season, and two playoff pairs have been confirmed. The Los Angeles Lakers beat the New Orleans Pelicans, 110-106, in the Play-In Tournament to secure the 7th seed to set up a first-round matchup with the Denver Nuggets. Meanwhile, the Sacramento Kings will host the Golden State Warriors in the second Western Conference NBA Play-In Tournament game. The winners secure the No. 8 seed in the NBA playoffs for its conference. EuroLeague Play-In: Baskonia-Virtus game schedule announced\", \"score\": 0.8479807, \"raw_content\": null}, {\"title\": \"NBA Finals 2024 - Celtics-Mavericks news, schedule, scores and ... - ESPN\", \"url\": \"https://www.espn.com/nba/story/_/id/39943302/nba-playoffs-2024-conference-finals-news-scores-highlights\", \"content\": \"The Boston Celtics are the 2024 NBA Champions. ... Western Conference. Final 2023-24 NBA regular-season standings. Which team left standing has the most trips to the NBA Finals? Here is a look at\", \"score\": 0.81979275, \"raw_content\": null}]}\u001b[0m\n", + "\u001b[33minference> \u001b[0m\u001b[33mThe\u001b[0m\u001b[33m teams\u001b[0m\u001b[33m that\u001b[0m\u001b[33m played\u001b[0m\u001b[33m in\u001b[0m\u001b[33m the\u001b[0m\u001b[33m NBA\u001b[0m\u001b[33m Western\u001b[0m\u001b[33m Conference\u001b[0m\u001b[33m Finals\u001b[0m\u001b[33m of\u001b[0m\u001b[33m \u001b[0m\u001b[33m202\u001b[0m\u001b[33m4\u001b[0m\u001b[33m were\u001b[0m\u001b[33m the\u001b[0m\u001b[33m Dallas\u001b[0m\u001b[33m Mavericks\u001b[0m\u001b[33m and\u001b[0m\u001b[33m the\u001b[0m\u001b[33m Minnesota\u001b[0m\u001b[33m Timber\u001b[0m\u001b[33mw\u001b[0m\u001b[33molves\u001b[0m\u001b[33m.\u001b[0m\u001b[97m\u001b[0m\n", + "\u001b[30m\u001b[0m\u001b[30m\u001b[0m\u001b[33minference> \u001b[0m\u001b[36m\u001b[0m\u001b[36mbr\u001b[0m\u001b[36mave\u001b[0m\u001b[36m_search\u001b[0m\u001b[36m.call\u001b[0m\u001b[36m(query\u001b[0m\u001b[36m=\"\u001b[0m\u001b[36mBill\u001b[0m\u001b[36m Cosby\u001b[0m\u001b[36m South\u001b[0m\u001b[36m Park\u001b[0m\u001b[36m episode\u001b[0m\u001b[36m\")\u001b[0m\u001b[97m\u001b[0m\n", + "\u001b[32mtool_execution> Tool:brave_search Args:{'query': 'Bill Cosby South Park episode'}\u001b[0m\n", + "\u001b[32mtool_execution> Tool:brave_search Response:{\"query\": \"Bill Cosby South Park episode\", \"top_k\": [{\"title\": \"Bill Cosby and Taylor Swift Duet - South Park Studios\", \"url\": \"https://www.southparkstudios.com/video-clips/90r7i1/south-park-bill-cosby-and-taylor-swift-duet\", \"content\": \"01:05 Bill Cosby is Here to See You South ParkS18 E10 ---------------------------------------------------- Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. 01:03 Bill Cosby and Taylor Swift Duet South ParkS18 E10 ------------------------------------------------------- The holiday special continues with Bill Cosby and Taylor Swift's rendition of \\\"It's Snowing Out There\\\". 01:31 #WeBelieveInYou South ParkS18 E10 -------------------------------------- With everyone watching, Kyle takes the opportunity to reach out to his brother. 01:47 Watch Your Microaggressions, Bro South ParkS19 E1 ------------------------------------------------------ Cartman's plan to frame PC Principal backfires. South ParkS19 E1 -------------------------------------- After hearing that the PC people have targeted Kyle, Cartman vows to help.\", \"score\": 0.685971, \"raw_content\": null}, {\"title\": \"Bill Cosby is Here to See You - South Park Studios US\", \"url\": \"https://southpark.cc.com/video-clips/wfot8s/south-park-bill-cosby-is-here-to-see-you\", \"content\": \"01:56 It's Not About Music South ParkS18 E9 ------------------------------------------ At home, Randy sees the consequences of Lorde's performance and calls the Record Producer to try and fix it. 01:24 Lorde's Hologram South ParkS18 E9 -------------------------------------- The Record Producer reveals the truth about the music industry... South ParkS18 E9 --------------------------------------------- Randy catches Sharon with Tupac's hologram. 01:37 I've Got Your Son, Lorde South ParkS18 E10 ----------------------------------------------- The Record Producer takes Stan and Kyle hostage. 01:05 Bill Cosby is Here to See You South ParkS18 E10 ---------------------------------------------------- Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. 01:21 Lorde Is My Dad South ParkS18 E10 -------------------------------------- After trying to confront Cartman Bra, Stan finally reveals the truth about his dad.\", \"score\": 0.6643884, \"raw_content\": null}, {\"title\": \"Bill Cosby (android) | South Park Character ... - South Park Studios US\", \"url\": \"https://southpark.cc.com/wiki/Bill_Cosby_(android)\", \"content\": \"Bill Cosby (android) | South Park Character / Location / User talk etc | Official South Park Studios Wiki Sent back in time to destroy Eric Cartman's Dawson's Creek Trapper Keeper before it manifests into an omnipotent supercomputer that can destroy all humanity, \\\"Bill Cosby\\\" is really VSM471, an android or cyborg of some kind engineered by 'hoomans' in the distant future. He fails in his initial missions to infiltrate South Park Elementary's 4th Grade class, destroy the Trapper Keeper or Cartman himself, but with Stan Marsh and Kyle Broflovski's aid, he is able to succeed in preventing his dismal future, and painfully fades from existence. South Park and all related titles, logos and characters are trademarks of Comedy Partners.\", \"score\": 0.5052006, \"raw_content\": null}, {\"title\": \"\\\"South Park\\\" Clubhouses (TV Episode 1998) - IMDb\", \"url\": \"https://www.imdb.com/title/tt0705915/characters/nm0005295\", \"content\": \"\\\"South Park\\\" Clubhouses (TV Episode 1998) - Trey Parker as Stan Marsh, Eric Cartman, Phillip, Randy Marsh, Fat Abbot, Mr. Garrison, Mr. Mackey, 3rd Fat Abbot character, Roy, Teenage Boy #1, Clyde, Bill Cosby, Teenage Boy #2 - IMDb Awards & Events Trey Parker: Stan Marsh, Eric Cartman, Phillip, Randy Marsh, Fat Abbot, Mr. Garrison, Mr. Mackey, 3rd Fat Abbot character, Roy, Teenage Boy #1, Clyde, Bill Cosby, Teenage Boy #2 Mr. Garrison : Stan, are you paying attention? Stan : Yes, Mr. Garrison. Stan Marsh : Dare. Stan Marsh : What? Release Dates | Official Sites | Company Credits | Filming & Production | Technical Specs Photo & Video User Lists Related lists from IMDb users 2024 Watched TV Shows\", \"score\": 0.4604593, \"raw_content\": null}, {\"title\": \"Trapper Keeper (South Park) - Wikipedia\", \"url\": \"https://en.wikipedia.org/wiki/Trapper_Keeper_(South_Park)\", \"content\": \"\\\"Trapper Keeper\\\" is the twelfth episode of the fourth season of the animated television series South Park, and the 60th episode of the series overall. In the episode, a man from the future wants Cartman's new Trapper Keeper, while Mr. Garrison's kindergarten class holds an election for class president with confusing results. It is one of the many South Park episodes that parodies a current event.[1] The main plot of the episode involving the Trapper Keeper was written before the election,[1] but the subplot is a parody of the controversy surrounding the election's outcome.[2] \\\"Trapper Keeper\\\" did not originally feature the election storyline, only a subplot about Ike attending his first day of kindergarten.[3] \\\"Trapper Keeper\\\" Full episode at South Park Studios\", \"score\": 0.3839421, \"raw_content\": null}]}\u001b[0m\n", + "\u001b[33minference> \u001b[0m\u001b[33mBill\u001b[0m\u001b[33m Cosby\u001b[0m\u001b[33m (\u001b[0m\u001b[33mBS\u001b[0m\u001b[33mM\u001b[0m\u001b[33m-\u001b[0m\u001b[33m471\u001b[0m\u001b[33m)\u001b[0m\u001b[33m first\u001b[0m\u001b[33m appears\u001b[0m\u001b[33m in\u001b[0m\u001b[33m the\u001b[0m\u001b[33m episode\u001b[0m\u001b[33m \"\u001b[0m\u001b[33mTr\u001b[0m\u001b[33mapper\u001b[0m\u001b[33m Keeper\u001b[0m\u001b[33m\"\u001b[0m\u001b[33m (\u001b[0m\u001b[33mSeason\u001b[0m\u001b[33m \u001b[0m\u001b[33m4\u001b[0m\u001b[33m,\u001b[0m\u001b[33m Episode\u001b[0m\u001b[33m \u001b[0m\u001b[33m12\u001b[0m\u001b[33m)\u001b[0m\u001b[33m of\u001b[0m\u001b[33m South\u001b[0m\u001b[33m Park\u001b[0m\u001b[33m.\u001b[0m\u001b[97m\u001b[0m\n", + "\u001b[30m\u001b[0m\u001b[30m\u001b[0m\u001b[33minference> \u001b[0m\u001b[36m\u001b[0m\u001b[36mbr\u001b[0m\u001b[36mave\u001b[0m\u001b[36m_search\u001b[0m\u001b[36m.call\u001b[0m\u001b[36m(query\u001b[0m\u001b[36m=\"\u001b[0m\u001b[36mAndrew\u001b[0m\u001b[36m Tate\u001b[0m\u001b[36m kick\u001b[0m\u001b[36mboxing\u001b[0m\u001b[36m name\u001b[0m\u001b[36m\")\u001b[0m\u001b[97m\u001b[0m\n", + "\u001b[32mtool_execution> Tool:brave_search Args:{'query': 'Andrew Tate kickboxing name'}\u001b[0m\n", + "\u001b[32mtool_execution> Tool:brave_search Response:{\"query\": \"Andrew Tate kickboxing name\", \"top_k\": [{\"title\": \"Andrew Tate Age, Height, Weight, Family, Parents, Biography, Net Worth\", \"url\": \"https://biographywallah.com/andrew-tate-biography/\", \"content\": \"Andrew Tate Age, Height, Weight, Family, Parents, Biography, Net Worth \\u00bb Biography Wallah Andrew Tate Age, Height, Weight, Family, Parents, Biography, Net Worth Andrew Tate Biography NameAndrew TateReal nameEmory Andrew Tate IIIProfession \\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0\\u00a0Kickboxer, Commentator and BusinessmanDate of birth14 December 1986BirthplaceWashington D.C., United StatesAndrew Tate Age37 years old (as of 2024)NationalityBritish-AmericanZodiac SignSagittariusGenderMaleSchoolLocal School in Washington D.C., United StatesGirlfriend/SpouseNaghel GeorgianaSexual OrientationStraightNet worth$1000 Million Who is Andrew Tate? Andrew Tate is a British-American former professional kickboxing world champion businessman and media personality, who fought in the cruiserweight and super cruiserweight divisions. Andrew Tate Age Andrew Tate was born on 1 December 1986 and is 37 years old. Andrew Tate\\u2019s Net Worth What is the net worth of Andrew Tate? Where is Andrew Tate from? How old is Andrew Tate?\", \"score\": 0.80698997, \"raw_content\": null}, {\"title\": \"The Life Of Andrew Tate (By Andrew Tate Himself ... - Sidekick Boxing\", \"url\": \"https://sidekickboxing.co.uk/the-life-of-andrew-king-cobra-tate/\", \"content\": \"Andrew Tate is a British-American former professional kickboxing world champion who fought in the cruiserweight and super cruiserweight divisions. Andrew Tate\\u2019s Kickboxing Career Andrew Tate in the Big Brother house Andrew Tate\\u2019s Kickboxing World Titles and his Sidekick boxing gloves Andrew Tate After Kickboxing Andrew Tate and his brother Tristan moved to Romania to set up their empire of businesses including trading in Bitcoin, Hustlers University, CobraTate.com, The Real World, and The War Room. From being a 4x kickboxing world champion to becoming the world\\u2019s most Googled man in the world with a private jet and over 33 cars, Andrew Tate\\u2019s life has been full of adventure.\", \"score\": 0.78194773, \"raw_content\": null}, {\"title\": \"Andrew Tate (\\\"King Cobra\\\") | MMA Fighter Page - Tapology\", \"url\": \"https://www.tapology.com/fightcenter/fighters/72139-andrew-tate\", \"content\": \"Andrew Tate (\\\"King Cobra\\\") | MMA Fighter Page | Tapology Andrew \\\"King Cobra\\\" Tate Andrew Tate Name: Andrew Tate Height: 6'1\\\" (185cm) | Reach: Andrew Tate is ineligible for Tapology's regional MMA rankings due to inactivity. Fighters must have at least one completed MMA bout in the past two years to be ranked. Andrew Tate MMA Fight Record Former top-ranked UFC fighter has called out Andrew Tate for having a paper title when it comes to combat... Andrew Tate \\u2022 All the biggest upcoming MMA & Boxing fights | UFC Fight Night | 02.01.2025, 12:00 PM ET | MMA Junkie: UFC Fight Night 249 video: Nine stoppages to open the year?! MMA Mania: Prochazka Vs. Hill: Odds, Full Fight Preview & Prediction\", \"score\": 0.6999322, \"raw_content\": null}, {\"title\": \"About Andrew Tate: A Journey from Champion to Controversy\", \"url\": \"https://reachmorpheus.com/andrew-tate/\", \"content\": \"Andrew Tate's kickboxing career, beginning in 2005, is a tale of determination and skill. He quickly made a name for himself in the sport, rising through the ranks with his unique fighting style and strategic approach, honed by his chess-playing background.\", \"score\": 0.6490677, \"raw_content\": null}, {\"title\": \"Andrew Tate's Kickboxing Career & Biography - MMA Full Contact\", \"url\": \"https://www.mmafullcontact.com/andrew-tate-kickboxing/\", \"content\": \"Andrew Tate's Kickboxing Career & Biography - MMA Full Contact Andrew Tate\\u2019s Kickboxing Career & Biography 2 Notable Opponents and Fights in Andrew Tate\\u2019s Kickboxing Career 4 Will Andrew Tate fight KSI? Notable Opponents and Fights in Andrew Tate\\u2019s Kickboxing Career Will Andrew Tate fight KSI? Similarly, Andrew Tate, known for his successful kickboxing career, has also shown interest in a potential fight with KSI. In conclusion, while there\\u2019s been plenty of interest and discussion about a potential boxing match between KSI and Andrew Tate, no official confirmation has been made as of now. With KSI\\u2019s upcoming match and Tate\\u2019s current personal circumstances, fans and followers of both personalities will have to wait for more updates on this potential fight.\", \"score\": 0.53050464, \"raw_content\": null}]}\u001b[0m\n", + "\u001b[33minference> \u001b[0m\u001b[33mAndrew\u001b[0m\u001b[33m Tate\u001b[0m\u001b[33m's\u001b[0m\u001b[33m kick\u001b[0m\u001b[33mboxing\u001b[0m\u001b[33m name\u001b[0m\u001b[33m is\u001b[0m\u001b[33m \"\u001b[0m\u001b[33mKing\u001b[0m\u001b[33m Cobra\u001b[0m\u001b[33m.\"\u001b[0m\u001b[97m\u001b[0m\n", + "\u001b[30m\u001b[0m" ] } ], "source": [ + "# NBVAL_SKIP\n", "from llama_stack_client.lib.agents.agent import Agent\n", "from llama_stack_client.lib.agents.event_logger import EventLogger\n", "from llama_stack_client.types.agent_create_params import AgentConfig\n", - "from google.colab import userdata\n", "\n", "agent_config = AgentConfig(\n", - " model=\"meta-llama/Llama-3.1-405B-Instruct\",\n", + " model=\"meta-llama/Llama-3.1-405B-Instruct-FP8\",\n", " instructions=\"You are a helpful assistant. Use search tool to answer the questions. \",\n", - " tools=(\n", - " [\n", - " {\n", - " \"type\": \"brave_search\",\n", - " \"engine\": \"tavily\",\n", - " \"api_key\": userdata.get(\"TAVILY_SEARCH_API_KEY\")\n", - " }\n", - " ]\n", - " ),\n", + " toolgroups=[\"builtin::websearch\"],\n", " input_shields=[],\n", " output_shields=[],\n", " enable_session_persistence=False,\n", @@ -2125,7 +3305,7 @@ " )\n", "\n", " for log in EventLogger().log(response):\n", - " log.print()" + " log.print()\n" ] }, { @@ -2140,22 +3320,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "id": "agkWgToGAsuA", "metadata": { "colab": { "base_uri": "https://localhost:8080/", - "height": 760 + "height": 1000 }, "id": "agkWgToGAsuA", - "outputId": "647cd5d2-7610-4fd6-ef66-c3f2f782a1b0" + "outputId": "4233a1d9-8282-4aa9-bdc4-0c105939f97e" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Getting traces for session_id=ac651ce8-2281-47f2-8814-ef947c066e40\n" + "Getting traces for session_id=72993b3e-6030-44f5-9f48-664449d2b6d3\n" ] }, { @@ -2167,42 +3347,82 @@ "│ │ │ '{\"role\":\"system\",\"content\":\"You are a helpful assistant. Use search tool to answer the questions. \"}',\n", "│ │ │ '{\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null}'\n", "│ │ ],\n", - "│ │ 'output': 'content: Let me check the latest sports news. tool_calls: []'\n", - "│ },\n", - "│ {\n", - "│ │ 'input': [\n", - "│ │ │ '{\"role\":\"system\",\"content\":\"You are a helpful assistant. Use search tool to answer the questions. \"}',\n", - "│ │ │ '{\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null}',\n", - "│ │ │ '{\"role\":\"assistant\",\"content\":\"Let me check the latest sports news.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[]}',\n", + "│ │ 'output': \"content: tool_calls: [ToolCall(call_id='8b7294ec-a83f-4798-ad8f-6bed662f08b6', tool_name=<BuiltinTool.brave_search: 'brave_search'>, arguments={'query': 'NBA Western Conference Finals 2024 teams'})]\"\n", + "│ },\n", + "│ {\n", + "│ │ 'input': '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"8b7294ec-a83f-4798-ad8f-6bed662f08b6\",\"tool_name\":\"brave_search\",\"arguments\":{\"query\":\"NBA Western Conference Finals 2024 teams\"}}]}',\n", + "│ │ 'output': '{\"role\":\"tool\",\"call_id\":\"8b7294ec-a83f-4798-ad8f-6bed662f08b6\",\"tool_name\":\"brave_search\",\"content\":\"{\\\\\"query\\\\\": \\\\\"NBA Western Conference Finals 2024 teams\\\\\", \\\\\"top_k\\\\\": [{\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference Finals - Basketball-Reference.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.basketball-reference.com/playoffs/2024-nba-western-conference-finals-mavericks-vs-timberwolves.html\\\\\", \\\\\"content\\\\\": \\\\\"2024 NBA Western Conference Finals Mavericks vs. Timberwolves League Champion: Boston Celtics. Finals MVP: Jaylen Brown (20.8 / 5.4 / 5.0) 2024 Playoff Leaders: PTS: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (635) TRB: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (208) AST: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (178) WS: Derrick White (2.9) More playoffs info\\\\\", \\\\\"score\\\\\": 0.9310187, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates, schedule and more - Sportskeeda\\\\\", \\\\\"url\\\\\": \\\\\"https://www.sportskeeda.com/basketball/news-nba-western-conference-finals-2024-dates-schedule-and-more\\\\\", \\\\\"content\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates & Schedule The 2023-24 NBA Western Conference Finals will start on Wednesday, May 22. The Mavericks will face the team that wins in Game 7 between the\\\\\", \\\\\"score\\\\\": 0.8914433, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"2024 Playoffs: West Finals | Timberwolves (3) vs. Mavericks (5) - NBA.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.nba.com/playoffs/2024/west-final\\\\\", \\\\\"content\\\\\": \\\\\"The Dallas Mavericks and Minnesota Timberwolves have advanced to the 2024 Western Conference Finals during the NBA playoffs.\\\\\", \\\\\"score\\\\\": 0.8884594, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference playoff bracket - Basketnews.com\\\\\", \\\\\"url\\\\\": \\\\\"https://basketnews.com/news-204687-2024-nba-western-conference-playoff-bracket.html\\\\\", \\\\\"content\\\\\": \\\\\"In the 2024 NBA Western Conference playoffs, the Oklahoma City Thunder clinched the No. 1 seed. Every team from the Western Conference played their final game of the regular season, and two playoff pairs have been confirmed. The Los Angeles Lakers beat the New Orleans Pelicans, 110-106, in the Play-In Tournament to secure the 7th seed to set up a first-round matchup with the Denver Nuggets. Meanwhile, the Sacramento Kings will host the Golden State Warriors in the second Western Conference NBA Play-In Tournament game. The winners secure the No. 8 seed in the NBA playoffs for its conference. EuroLeague Play-In: Baskonia-Virtus game schedule announced\\\\\", \\\\\"score\\\\\": 0.8479807, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"NBA Finals 2024 - Celtics-Mavericks news, schedule, scores and ... - ESPN\\\\\", \\\\\"url\\\\\": \\\\\"https://www.espn.com/nba/story/_/id/39943302/nba-playoffs-2024-conference-finals-news-scores-highlights\\\\\", \\\\\"content\\\\\": \\\\\"The Boston Celtics are the 2024 NBA Champions. ... Western Conference. Final 2023-24 NBA regular-season standings. Which team left standing has the most trips to the NBA Finals? Here is a look at\\\\\", \\\\\"score\\\\\": 0.81979275, \\\\\"raw_content\\\\\": null}]}\"}'\n", + "│ },\n", + "│ {\n", + "│ │ 'input': [\n", + "│ │ │ '{\"role\":\"system\",\"content\":\"You are a helpful assistant. Use search tool to answer the questions. \"}',\n", + "│ │ │ '{\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null}',\n", + "│ │ │ '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"8b7294ec-a83f-4798-ad8f-6bed662f08b6\",\"tool_name\":\"brave_search\",\"arguments\":{\"query\":\"NBA Western Conference Finals 2024 teams\"}}]}',\n", + "│ │ │ '{\"role\":\"tool\",\"call_id\":\"8b7294ec-a83f-4798-ad8f-6bed662f08b6\",\"tool_name\":\"brave_search\",\"content\":\"{\\\\\"query\\\\\": \\\\\"NBA Western Conference Finals 2024 teams\\\\\", \\\\\"top_k\\\\\": [{\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference Finals - Basketball-Reference.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.basketball-reference.com/playoffs/2024-nba-western-conference-finals-mavericks-vs-timberwolves.html\\\\\", \\\\\"content\\\\\": \\\\\"2024 NBA Western Conference Finals Mavericks vs. Timberwolves League Champion: Boston Celtics. Finals MVP: Jaylen Brown (20.8 / 5.4 / 5.0) 2024 Playoff Leaders: PTS: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (635) TRB: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (208) AST: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (178) WS: Derrick White (2.9) More playoffs info\\\\\", \\\\\"score\\\\\": 0.9310187, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates, schedule and more - Sportskeeda\\\\\", \\\\\"url\\\\\": \\\\\"https://www.sportskeeda.com/basketball/news-nba-western-conference-finals-2024-dates-schedule-and-more\\\\\", \\\\\"content\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates & Schedule The 2023-24 NBA Western Conference Finals will start on Wednesday, May 22. The Mavericks will face the team that wins in Game 7 between the\\\\\", \\\\\"score\\\\\": 0.8914433, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"2024 Playoffs: West Finals | Timberwolves (3) vs. Mavericks (5) - NBA.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.nba.com/playoffs/2024/west-final\\\\\", \\\\\"content\\\\\": \\\\\"The Dallas Mavericks and Minnesota Timberwolves have advanced to the 2024 Western Conference Finals during the NBA playoffs.\\\\\", \\\\\"score\\\\\": 0.8884594, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference playoff bracket - Basketnews.com\\\\\", \\\\\"url\\\\\": \\\\\"https://basketnews.com/news-204687-2024-nba-western-conference-playoff-bracket.html\\\\\", \\\\\"content\\\\\": \\\\\"In the 2024 NBA Western Conference playoffs, the Oklahoma City Thunder clinched the No. 1 seed. Every team from the Western Conference played their final game of the regular season, and two playoff pairs have been confirmed. The Los Angeles Lakers beat the New Orleans Pelicans, 110-106, in the Play-In Tournament to secure the 7th seed to set up a first-round matchup with the Denver Nuggets. Meanwhile, the Sacramento Kings will host the Golden State Warriors in the second Western Conference NBA Play-In Tournament game. The winners secure the No. 8 seed in the NBA playoffs for its conference. EuroLeague Play-In: Baskonia-Virtus game schedule announced\\\\\", \\\\\"score\\\\\": 0.8479807, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"NBA Finals 2024 - Celtics-Mavericks news, schedule, scores and ... - ESPN\\\\\", \\\\\"url\\\\\": \\\\\"https://www.espn.com/nba/story/_/id/39943302/nba-playoffs-2024-conference-finals-news-scores-highlights\\\\\", \\\\\"content\\\\\": \\\\\"The Boston Celtics are the 2024 NBA Champions. ... Western Conference. Final 2023-24 NBA regular-season standings. Which team left standing has the most trips to the NBA Finals? Here is a look at\\\\\", \\\\\"score\\\\\": 0.81979275, \\\\\"raw_content\\\\\": null}]}\"}'\n", + "│ │ ],\n", + "│ │ 'output': 'content: The teams that played in the NBA Western Conference Finals of 2024 were the Dallas Mavericks and the Minnesota Timberwolves. tool_calls: []'\n", + "│ },\n", + "│ {\n", + "│ │ 'input': [\n", + "│ │ │ '{\"role\":\"system\",\"content\":\"You are a helpful assistant. Use search tool to answer the questions. \"}',\n", + "│ │ │ '{\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null}',\n", + "│ │ │ '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"8b7294ec-a83f-4798-ad8f-6bed662f08b6\",\"tool_name\":\"brave_search\",\"arguments\":{\"query\":\"NBA Western Conference Finals 2024 teams\"}}]}',\n", + "│ │ │ '{\"role\":\"tool\",\"call_id\":\"8b7294ec-a83f-4798-ad8f-6bed662f08b6\",\"tool_name\":\"brave_search\",\"content\":\"{\\\\\"query\\\\\": \\\\\"NBA Western Conference Finals 2024 teams\\\\\", \\\\\"top_k\\\\\": [{\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference Finals - Basketball-Reference.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.basketball-reference.com/playoffs/2024-nba-western-conference-finals-mavericks-vs-timberwolves.html\\\\\", \\\\\"content\\\\\": \\\\\"2024 NBA Western Conference Finals Mavericks vs. Timberwolves League Champion: Boston Celtics. Finals MVP: Jaylen Brown (20.8 / 5.4 / 5.0) 2024 Playoff Leaders: PTS: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (635) TRB: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (208) AST: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (178) WS: Derrick White (2.9) More playoffs info\\\\\", \\\\\"score\\\\\": 0.9310187, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates, schedule and more - Sportskeeda\\\\\", \\\\\"url\\\\\": \\\\\"https://www.sportskeeda.com/basketball/news-nba-western-conference-finals-2024-dates-schedule-and-more\\\\\", \\\\\"content\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates & Schedule The 2023-24 NBA Western Conference Finals will start on Wednesday, May 22. The Mavericks will face the team that wins in Game 7 between the\\\\\", \\\\\"score\\\\\": 0.8914433, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"2024 Playoffs: West Finals | Timberwolves (3) vs. Mavericks (5) - NBA.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.nba.com/playoffs/2024/west-final\\\\\", \\\\\"content\\\\\": \\\\\"The Dallas Mavericks and Minnesota Timberwolves have advanced to the 2024 Western Conference Finals during the NBA playoffs.\\\\\", \\\\\"score\\\\\": 0.8884594, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference playoff bracket - Basketnews.com\\\\\", \\\\\"url\\\\\": \\\\\"https://basketnews.com/news-204687-2024-nba-western-conference-playoff-bracket.html\\\\\", \\\\\"content\\\\\": \\\\\"In the 2024 NBA Western Conference playoffs, the Oklahoma City Thunder clinched the No. 1 seed. Every team from the Western Conference played their final game of the regular season, and two playoff pairs have been confirmed. The Los Angeles Lakers beat the New Orleans Pelicans, 110-106, in the Play-In Tournament to secure the 7th seed to set up a first-round matchup with the Denver Nuggets. Meanwhile, the Sacramento Kings will host the Golden State Warriors in the second Western Conference NBA Play-In Tournament game. The winners secure the No. 8 seed in the NBA playoffs for its conference. EuroLeague Play-In: Baskonia-Virtus game schedule announced\\\\\", \\\\\"score\\\\\": 0.8479807, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"NBA Finals 2024 - Celtics-Mavericks news, schedule, scores and ... - ESPN\\\\\", \\\\\"url\\\\\": \\\\\"https://www.espn.com/nba/story/_/id/39943302/nba-playoffs-2024-conference-finals-news-scores-highlights\\\\\", \\\\\"content\\\\\": \\\\\"The Boston Celtics are the 2024 NBA Champions. ... Western Conference. Final 2023-24 NBA regular-season standings. Which team left standing has the most trips to the NBA Finals? Here is a look at\\\\\", \\\\\"score\\\\\": 0.81979275, \\\\\"raw_content\\\\\": null}]}\"}',\n", + "│ │ │ '{\"role\":\"assistant\",\"content\":\"The teams that played in the NBA Western Conference Finals of 2024 were the Dallas Mavericks and the Minnesota Timberwolves.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[]}',\n", "│ │ │ '{\"role\":\"user\",\"content\":\"In which episode and season of South Park does Bill Cosby (BSM-471) first appear? Give me the number and title.\",\"context\":null}'\n", - "│ │ ],\n", - "│ │ 'output': \"content: tool_calls: [ToolCall(call_id='19bd3554-e670-4856-89d0-c63f5b016245', tool_name='bravy_search', arguments={'query': 'Bill Cosby South Park episode'})]\"\n", - "│ },\n", - "│ {\n", - "│ │ 'input': [\n", - "│ │ │ '{\"role\":\"system\",\"content\":\"You are a helpful assistant. Use search tool to answer the questions. \"}',\n", - "│ │ │ '{\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null}',\n", - "│ │ │ '{\"role\":\"assistant\",\"content\":\"Let me check the latest sports news.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[]}',\n", - "│ │ │ '{\"role\":\"user\",\"content\":\"In which episode and season of South Park does Bill Cosby (BSM-471) first appear? Give me the number and title.\",\"context\":null}',\n", - "│ │ │ '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"19bd3554-e670-4856-89d0-c63f5b016245\",\"tool_name\":\"bravy_search\",\"arguments\":{\"query\":\"Bill Cosby South Park episode\"}}]}',\n", + "│ │ ],\n", + "│ │ 'output': \"content: tool_calls: [ToolCall(call_id='fc0441bf-05ad-48d0-8034-4e19cb835904', tool_name=<BuiltinTool.brave_search: 'brave_search'>, arguments={'query': 'Bill Cosby South Park episode'})]\"\n", + "│ },\n", + "│ {\n", + "│ │ 'input': '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"fc0441bf-05ad-48d0-8034-4e19cb835904\",\"tool_name\":\"brave_search\",\"arguments\":{\"query\":\"Bill Cosby South Park episode\"}}]}',\n", + "│ │ 'output': '{\"role\":\"tool\",\"call_id\":\"fc0441bf-05ad-48d0-8034-4e19cb835904\",\"tool_name\":\"brave_search\",\"content\":\"{\\\\\"query\\\\\": \\\\\"Bill Cosby South Park episode\\\\\", \\\\\"top_k\\\\\": [{\\\\\"title\\\\\": \\\\\"Bill Cosby and Taylor Swift Duet - South Park Studios\\\\\", \\\\\"url\\\\\": \\\\\"https://www.southparkstudios.com/video-clips/90r7i1/south-park-bill-cosby-and-taylor-swift-duet\\\\\", \\\\\"content\\\\\": \\\\\"01:05 Bill Cosby is Here to See You South ParkS18 E10 ---------------------------------------------------- Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. 01:03 Bill Cosby and Taylor Swift Duet South ParkS18 E10 ------------------------------------------------------- The holiday special continues with Bill Cosby and Taylor Swift\\'s rendition of \\\\\\\\\\\\\"It\\'s Snowing Out There\\\\\\\\\\\\\". 01:31 #WeBelieveInYou South ParkS18 E10 -------------------------------------- With everyone watching, Kyle takes the opportunity to reach out to his brother. 01:47 Watch Your Microaggressions, Bro South ParkS19 E1 ------------------------------------------------------ Cartman\\'s plan to frame PC Principal backfires. South ParkS19 E1 -------------------------------------- After hearing that the PC people have targeted Kyle, Cartman vows to help.\\\\\", \\\\\"score\\\\\": 0.685971, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Bill Cosby is Here to See You - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/video-clips/wfot8s/south-park-bill-cosby-is-here-to-see-you\\\\\", \\\\\"content\\\\\": \\\\\"01:56 It\\'s Not About Music South ParkS18 E9 ------------------------------------------ At home, Randy sees the consequences of Lorde\\'s performance and calls the Record Producer to try and fix it. 01:24 Lorde\\'s Hologram South ParkS18 E9 -------------------------------------- The Record Producer reveals the truth about the music industry... South ParkS18 E9 --------------------------------------------- Randy catches Sharon with Tupac\\'s hologram. 01:37 I\\'ve Got Your Son, Lorde South ParkS18 E10 ----------------------------------------------- The Record Producer takes Stan and Kyle hostage. 01:05 Bill Cosby is Here to See You South ParkS18 E10 ---------------------------------------------------- Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. 01:21 Lorde Is My Dad South ParkS18 E10 -------------------------------------- After trying to confront Cartman Bra, Stan finally reveals the truth about his dad.\\\\\", \\\\\"score\\\\\": 0.6643884, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Bill Cosby (android) | South Park Character ... - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/wiki/Bill_Cosby_(android)\\\\\", \\\\\"content\\\\\": \\\\\"Bill Cosby (android) | South Park Character / Location / User talk etc | Official South Park Studios Wiki Sent back in time to destroy Eric Cartman\\'s Dawson\\'s Creek Trapper Keeper before it manifests into an omnipotent supercomputer that can destroy all humanity, \\\\\\\\\\\\\"Bill Cosby\\\\\\\\\\\\\" is really VSM471, an android or cyborg of some kind engineered by \\'hoomans\\' in the distant future. He fails in his initial missions to infiltrate South Park Elementary\\'s 4th Grade class, destroy the Trapper Keeper or Cartman himself, but with Stan Marsh and Kyle Broflovski\\'s aid, he is able to succeed in preventing his dismal future, and painfully fades from existence. South Park and all related titles, logos and characters are trademarks of Comedy Partners.\\\\\", \\\\\"score\\\\\": 0.5052006, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"\\\\\\\\\\\\\"South Park\\\\\\\\\\\\\" Clubhouses (TV Episode 1998) - IMDb\\\\\", \\\\\"url\\\\\": \\\\\"https://www.imdb.com/title/tt0705915/characters/nm0005295\\\\\", \\\\\"content\\\\\": \\\\\"\\\\\\\\\\\\\"South Park\\\\\\\\\\\\\" Clubhouses (TV Episode 1998) - Trey Parker as Stan Marsh, Eric Cartman, Phillip, Randy Marsh, Fat Abbot, Mr. Garrison, Mr. Mackey, 3rd Fat Abbot character, Roy, Teenage Boy #1, Clyde, Bill Cosby, Teenage Boy #2 - IMDb Awards & Events Trey Parker: Stan Marsh, Eric Cartman, Phillip, Randy Marsh, Fat Abbot, Mr. Garrison, Mr. Mackey, 3rd Fat Abbot character, Roy, Teenage Boy #1, Clyde, Bill Cosby, Teenage Boy #2 Mr. Garrison : Stan, are you paying attention? Stan : Yes, Mr. Garrison. Stan Marsh : Dare. Stan Marsh : What? Release Dates | Official Sites | Company Credits | Filming & Production | Technical Specs Photo & Video User Lists Related lists from IMDb users 2024 Watched TV Shows\\\\\", \\\\\"score\\\\\": 0.4604593, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Trapper Keeper (South Park) - Wikipedia\\\\\", \\\\\"url\\\\\": \\\\\"https://en.wikipedia.org/wiki/Trapper_Keeper_(South_Park)\\\\\", \\\\\"content\\\\\": \\\\\"\\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" is the twelfth episode of the fourth season of the animated television series South Park, and the 60th episode of the series overall. In the episode, a man from the future wants Cartman\\'s new Trapper Keeper, while Mr. Garrison\\'s kindergarten class holds an election for class president with confusing results. It is one of the many South Park episodes that parodies a current event.[1] The main plot of the episode involving the Trapper Keeper was written before the election,[1] but the subplot is a parody of the controversy surrounding the election\\'s outcome.[2] \\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" did not originally feature the election storyline, only a subplot about Ike attending his first day of kindergarten.[3] \\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" Full episode at South Park Studios\\\\\", \\\\\"score\\\\\": 0.3839421, \\\\\"raw_content\\\\\": null}]}\"}'\n", + "│ },\n", + "│ {\n", + "│ │ 'input': [\n", + "│ │ │ '{\"role\":\"system\",\"content\":\"You are a helpful assistant. Use search tool to answer the questions. \"}',\n", + "│ │ │ '{\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null}',\n", + "│ │ │ '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"8b7294ec-a83f-4798-ad8f-6bed662f08b6\",\"tool_name\":\"brave_search\",\"arguments\":{\"query\":\"NBA Western Conference Finals 2024 teams\"}}]}',\n", + "│ │ │ '{\"role\":\"tool\",\"call_id\":\"8b7294ec-a83f-4798-ad8f-6bed662f08b6\",\"tool_name\":\"brave_search\",\"content\":\"{\\\\\"query\\\\\": \\\\\"NBA Western Conference Finals 2024 teams\\\\\", \\\\\"top_k\\\\\": [{\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference Finals - Basketball-Reference.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.basketball-reference.com/playoffs/2024-nba-western-conference-finals-mavericks-vs-timberwolves.html\\\\\", \\\\\"content\\\\\": \\\\\"2024 NBA Western Conference Finals Mavericks vs. Timberwolves League Champion: Boston Celtics. Finals MVP: Jaylen Brown (20.8 / 5.4 / 5.0) 2024 Playoff Leaders: PTS: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (635) TRB: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (208) AST: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (178) WS: Derrick White (2.9) More playoffs info\\\\\", \\\\\"score\\\\\": 0.9310187, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates, schedule and more - Sportskeeda\\\\\", \\\\\"url\\\\\": \\\\\"https://www.sportskeeda.com/basketball/news-nba-western-conference-finals-2024-dates-schedule-and-more\\\\\", \\\\\"content\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates & Schedule The 2023-24 NBA Western Conference Finals will start on Wednesday, May 22. The Mavericks will face the team that wins in Game 7 between the\\\\\", \\\\\"score\\\\\": 0.8914433, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"2024 Playoffs: West Finals | Timberwolves (3) vs. Mavericks (5) - NBA.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.nba.com/playoffs/2024/west-final\\\\\", \\\\\"content\\\\\": \\\\\"The Dallas Mavericks and Minnesota Timberwolves have advanced to the 2024 Western Conference Finals during the NBA playoffs.\\\\\", \\\\\"score\\\\\": 0.8884594, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference playoff bracket - Basketnews.com\\\\\", \\\\\"url\\\\\": \\\\\"https://basketnews.com/news-204687-2024-nba-western-conference-playoff-bracket.html\\\\\", \\\\\"content\\\\\": \\\\\"In the 2024 NBA Western Conference playoffs, the Oklahoma City Thunder clinched the No. 1 seed. Every team from the Western Conference played their final game of the regular season, and two playoff pairs have been confirmed. The Los Angeles Lakers beat the New Orleans Pelicans, 110-106, in the Play-In Tournament to secure the 7th seed to set up a first-round matchup with the Denver Nuggets. Meanwhile, the Sacramento Kings will host the Golden State Warriors in the second Western Conference NBA Play-In Tournament game. The winners secure the No. 8 seed in the NBA playoffs for its conference. EuroLeague Play-In: Baskonia-Virtus game schedule announced\\\\\", \\\\\"score\\\\\": 0.8479807, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"NBA Finals 2024 - Celtics-Mavericks news, schedule, scores and ... - ESPN\\\\\", \\\\\"url\\\\\": \\\\\"https://www.espn.com/nba/story/_/id/39943302/nba-playoffs-2024-conference-finals-news-scores-highlights\\\\\", \\\\\"content\\\\\": \\\\\"The Boston Celtics are the 2024 NBA Champions. ... Western Conference. Final 2023-24 NBA regular-season standings. Which team left standing has the most trips to the NBA Finals? Here is a look at\\\\\", \\\\\"score\\\\\": 0.81979275, \\\\\"raw_content\\\\\": null}]}\"}',\n", + "│ │ │ '{\"role\":\"assistant\",\"content\":\"The teams that played in the NBA Western Conference Finals of 2024 were the Dallas Mavericks and the Minnesota Timberwolves.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[]}',\n", + "│ │ │ '{\"role\":\"user\",\"content\":\"In which episode and season of South Park does Bill Cosby (BSM-471) first appear? Give me the number and title.\",\"context\":null}',\n", + "│ │ │ '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"fc0441bf-05ad-48d0-8034-4e19cb835904\",\"tool_name\":\"brave_search\",\"arguments\":{\"query\":\"Bill Cosby South Park episode\"}}]}',\n", + "│ │ │ '{\"role\":\"tool\",\"call_id\":\"fc0441bf-05ad-48d0-8034-4e19cb835904\",\"tool_name\":\"brave_search\",\"content\":\"{\\\\\"query\\\\\": \\\\\"Bill Cosby South Park episode\\\\\", \\\\\"top_k\\\\\": [{\\\\\"title\\\\\": \\\\\"Bill Cosby and Taylor Swift Duet - South Park Studios\\\\\", \\\\\"url\\\\\": \\\\\"https://www.southparkstudios.com/video-clips/90r7i1/south-park-bill-cosby-and-taylor-swift-duet\\\\\", \\\\\"content\\\\\": \\\\\"01:05 Bill Cosby is Here to See You South ParkS18 E10 ---------------------------------------------------- Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. 01:03 Bill Cosby and Taylor Swift Duet South ParkS18 E10 ------------------------------------------------------- The holiday special continues with Bill Cosby and Taylor Swift\\'s rendition of \\\\\\\\\\\\\"It\\'s Snowing Out There\\\\\\\\\\\\\". 01:31 #WeBelieveInYou South ParkS18 E10 -------------------------------------- With everyone watching, Kyle takes the opportunity to reach out to his brother. 01:47 Watch Your Microaggressions, Bro South ParkS19 E1 ------------------------------------------------------ Cartman\\'s plan to frame PC Principal backfires. South ParkS19 E1 -------------------------------------- After hearing that the PC people have targeted Kyle, Cartman vows to help.\\\\\", \\\\\"score\\\\\": 0.685971, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Bill Cosby is Here to See You - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/video-clips/wfot8s/south-park-bill-cosby-is-here-to-see-you\\\\\", \\\\\"content\\\\\": \\\\\"01:56 It\\'s Not About Music South ParkS18 E9 ------------------------------------------ At home, Randy sees the consequences of Lorde\\'s performance and calls the Record Producer to try and fix it. 01:24 Lorde\\'s Hologram South ParkS18 E9 -------------------------------------- The Record Producer reveals the truth about the music industry... South ParkS18 E9 --------------------------------------------- Randy catches Sharon with Tupac\\'s hologram. 01:37 I\\'ve Got Your Son, Lorde South ParkS18 E10 ----------------------------------------------- The Record Producer takes Stan and Kyle hostage. 01:05 Bill Cosby is Here to See You South ParkS18 E10 ---------------------------------------------------- Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. 01:21 Lorde Is My Dad South ParkS18 E10 -------------------------------------- After trying to confront Cartman Bra, Stan finally reveals the truth about his dad.\\\\\", \\\\\"score\\\\\": 0.6643884, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Bill Cosby (android) | South Park Character ... - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/wiki/Bill_Cosby_(android)\\\\\", \\\\\"content\\\\\": \\\\\"Bill Cosby (android) | South Park Character / Location / User talk etc | Official South Park Studios Wiki Sent back in time to destroy Eric Cartman\\'s Dawson\\'s Creek Trapper Keeper before it manifests into an omnipotent supercomputer that can destroy all humanity, \\\\\\\\\\\\\"Bill Cosby\\\\\\\\\\\\\" is really VSM471, an android or cyborg of some kind engineered by \\'hoomans\\' in the distant future. He fails in his initial missions to infiltrate South Park Elementary\\'s 4th Grade class, destroy the Trapper Keeper or Cartman himself, but with Stan Marsh and Kyle Broflovski\\'s aid, he is able to succeed in preventing his dismal future, and painfully fades from existence. South Park and all related titles, logos and characters are trademarks of Comedy Partners.\\\\\", \\\\\"score\\\\\": 0.5052006, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"\\\\\\\\\\\\\"South Park\\\\\\\\\\\\\" Clubhouses (TV Episode 1998) - IMDb\\\\\", \\\\\"url\\\\\": \\\\\"https://www.imdb.com/title/tt0705915/characters/nm0005295\\\\\", \\\\\"content\\\\\": \\\\\"\\\\\\\\\\\\\"South Park\\\\\\\\\\\\\" Clubhouses (TV Episode 1998) - Trey Parker as Stan Marsh, Eric Cartman, Phillip, Randy Marsh, Fat Abbot, Mr. Garrison, Mr. Mackey, 3rd Fat Abbot character, Roy, Teenage Boy #1, Clyde, Bill Cosby, Teenage Boy #2 - IMDb Awards & Events Trey Parker: Stan Marsh, Eric Cartman, Phillip, Randy Marsh, Fat Abbot, Mr. Garrison, Mr. Mackey, 3rd Fat Abbot character, Roy, Teenage Boy #1, Clyde, Bill Cosby, Teenage Boy #2 Mr. Garrison : Stan, are you paying attention? Stan : Yes, Mr. Garrison. Stan Marsh : Dare. Stan Marsh : What? Release Dates | Official Sites | Company Credits | Filming & Production | Technical Specs Photo & Video User Lists Related lists from IMDb users 2024 Watched TV Shows\\\\\", \\\\\"score\\\\\": 0.4604593, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Trapper Keeper (South Park) - Wikipedia\\\\\", \\\\\"url\\\\\": \\\\\"https://en.wikipedia.org/wiki/Trapper_Keeper_(South_Park)\\\\\", \\\\\"content\\\\\": \\\\\"\\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" is the twelfth episode of the fourth season of the animated television series South Park, and the 60th episode of the series overall. In the episode, a man from the future wants Cartman\\'s new Trapper Keeper, while Mr. Garrison\\'s kindergarten class holds an election for class president with confusing results. It is one of the many South Park episodes that parodies a current event.[1] The main plot of the episode involving the Trapper Keeper was written before the election,[1] but the subplot is a parody of the controversy surrounding the election\\'s outcome.[2] \\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" did not originally feature the election storyline, only a subplot about Ike attending his first day of kindergarten.[3] \\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" Full episode at South Park Studios\\\\\", \\\\\"score\\\\\": 0.3839421, \\\\\"raw_content\\\\\": null}]}\"}'\n", + "│ │ ],\n", + "│ │ 'output': 'content: Bill Cosby (BSM-471) first appears in the episode \"Trapper Keeper\" (Season 4, Episode 12) of South Park. tool_calls: []'\n", + "│ },\n", + "│ {\n", + "│ │ 'input': [\n", + "│ │ │ '{\"role\":\"system\",\"content\":\"You are a helpful assistant. Use search tool to answer the questions. \"}',\n", + "│ │ │ '{\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null}',\n", + "│ │ │ '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"8b7294ec-a83f-4798-ad8f-6bed662f08b6\",\"tool_name\":\"brave_search\",\"arguments\":{\"query\":\"NBA Western Conference Finals 2024 teams\"}}]}',\n", + "│ │ │ '{\"role\":\"tool\",\"call_id\":\"8b7294ec-a83f-4798-ad8f-6bed662f08b6\",\"tool_name\":\"brave_search\",\"content\":\"{\\\\\"query\\\\\": \\\\\"NBA Western Conference Finals 2024 teams\\\\\", \\\\\"top_k\\\\\": [{\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference Finals - Basketball-Reference.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.basketball-reference.com/playoffs/2024-nba-western-conference-finals-mavericks-vs-timberwolves.html\\\\\", \\\\\"content\\\\\": \\\\\"2024 NBA Western Conference Finals Mavericks vs. Timberwolves League Champion: Boston Celtics. Finals MVP: Jaylen Brown (20.8 / 5.4 / 5.0) 2024 Playoff Leaders: PTS: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (635) TRB: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (208) AST: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (178) WS: Derrick White (2.9) More playoffs info\\\\\", \\\\\"score\\\\\": 0.9310187, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates, schedule and more - Sportskeeda\\\\\", \\\\\"url\\\\\": \\\\\"https://www.sportskeeda.com/basketball/news-nba-western-conference-finals-2024-dates-schedule-and-more\\\\\", \\\\\"content\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates & Schedule The 2023-24 NBA Western Conference Finals will start on Wednesday, May 22. The Mavericks will face the team that wins in Game 7 between the\\\\\", \\\\\"score\\\\\": 0.8914433, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"2024 Playoffs: West Finals | Timberwolves (3) vs. Mavericks (5) - NBA.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.nba.com/playoffs/2024/west-final\\\\\", \\\\\"content\\\\\": \\\\\"The Dallas Mavericks and Minnesota Timberwolves have advanced to the 2024 Western Conference Finals during the NBA playoffs.\\\\\", \\\\\"score\\\\\": 0.8884594, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference playoff bracket - Basketnews.com\\\\\", \\\\\"url\\\\\": \\\\\"https://basketnews.com/news-204687-2024-nba-western-conference-playoff-bracket.html\\\\\", \\\\\"content\\\\\": \\\\\"In the 2024 NBA Western Conference playoffs, the Oklahoma City Thunder clinched the No. 1 seed. Every team from the Western Conference played their final game of the regular season, and two playoff pairs have been confirmed. The Los Angeles Lakers beat the New Orleans Pelicans, 110-106, in the Play-In Tournament to secure the 7th seed to set up a first-round matchup with the Denver Nuggets. Meanwhile, the Sacramento Kings will host the Golden State Warriors in the second Western Conference NBA Play-In Tournament game. The winners secure the No. 8 seed in the NBA playoffs for its conference. EuroLeague Play-In: Baskonia-Virtus game schedule announced\\\\\", \\\\\"score\\\\\": 0.8479807, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"NBA Finals 2024 - Celtics-Mavericks news, schedule, scores and ... - ESPN\\\\\", \\\\\"url\\\\\": \\\\\"https://www.espn.com/nba/story/_/id/39943302/nba-playoffs-2024-conference-finals-news-scores-highlights\\\\\", \\\\\"content\\\\\": \\\\\"The Boston Celtics are the 2024 NBA Champions. ... Western Conference. Final 2023-24 NBA regular-season standings. Which team left standing has the most trips to the NBA Finals? Here is a look at\\\\\", \\\\\"score\\\\\": 0.81979275, \\\\\"raw_content\\\\\": null}]}\"}',\n", + "│ │ │ '{\"role\":\"assistant\",\"content\":\"The teams that played in the NBA Western Conference Finals of 2024 were the Dallas Mavericks and the Minnesota Timberwolves.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[]}',\n", + "│ │ │ '{\"role\":\"user\",\"content\":\"In which episode and season of South Park does Bill Cosby (BSM-471) first appear? Give me the number and title.\",\"context\":null}',\n", + "│ │ │ '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"fc0441bf-05ad-48d0-8034-4e19cb835904\",\"tool_name\":\"brave_search\",\"arguments\":{\"query\":\"Bill Cosby South Park episode\"}}]}',\n", + "│ │ │ '{\"role\":\"tool\",\"call_id\":\"fc0441bf-05ad-48d0-8034-4e19cb835904\",\"tool_name\":\"brave_search\",\"content\":\"{\\\\\"query\\\\\": \\\\\"Bill Cosby South Park episode\\\\\", \\\\\"top_k\\\\\": [{\\\\\"title\\\\\": \\\\\"Bill Cosby and Taylor Swift Duet - South Park Studios\\\\\", \\\\\"url\\\\\": \\\\\"https://www.southparkstudios.com/video-clips/90r7i1/south-park-bill-cosby-and-taylor-swift-duet\\\\\", \\\\\"content\\\\\": \\\\\"01:05 Bill Cosby is Here to See You South ParkS18 E10 ---------------------------------------------------- Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. 01:03 Bill Cosby and Taylor Swift Duet South ParkS18 E10 ------------------------------------------------------- The holiday special continues with Bill Cosby and Taylor Swift\\'s rendition of \\\\\\\\\\\\\"It\\'s Snowing Out There\\\\\\\\\\\\\". 01:31 #WeBelieveInYou South ParkS18 E10 -------------------------------------- With everyone watching, Kyle takes the opportunity to reach out to his brother. 01:47 Watch Your Microaggressions, Bro South ParkS19 E1 ------------------------------------------------------ Cartman\\'s plan to frame PC Principal backfires. South ParkS19 E1 -------------------------------------- After hearing that the PC people have targeted Kyle, Cartman vows to help.\\\\\", \\\\\"score\\\\\": 0.685971, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Bill Cosby is Here to See You - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/video-clips/wfot8s/south-park-bill-cosby-is-here-to-see-you\\\\\", \\\\\"content\\\\\": \\\\\"01:56 It\\'s Not About Music South ParkS18 E9 ------------------------------------------ At home, Randy sees the consequences of Lorde\\'s performance and calls the Record Producer to try and fix it. 01:24 Lorde\\'s Hologram South ParkS18 E9 -------------------------------------- The Record Producer reveals the truth about the music industry... South ParkS18 E9 --------------------------------------------- Randy catches Sharon with Tupac\\'s hologram. 01:37 I\\'ve Got Your Son, Lorde South ParkS18 E10 ----------------------------------------------- The Record Producer takes Stan and Kyle hostage. 01:05 Bill Cosby is Here to See You South ParkS18 E10 ---------------------------------------------------- Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. 01:21 Lorde Is My Dad South ParkS18 E10 -------------------------------------- After trying to confront Cartman Bra, Stan finally reveals the truth about his dad.\\\\\", \\\\\"score\\\\\": 0.6643884, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Bill Cosby (android) | South Park Character ... - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/wiki/Bill_Cosby_(android)\\\\\", \\\\\"content\\\\\": \\\\\"Bill Cosby (android) | South Park Character / Location / User talk etc | Official South Park Studios Wiki Sent back in time to destroy Eric Cartman\\'s Dawson\\'s Creek Trapper Keeper before it manifests into an omnipotent supercomputer that can destroy all humanity, \\\\\\\\\\\\\"Bill Cosby\\\\\\\\\\\\\" is really VSM471, an android or cyborg of some kind engineered by \\'hoomans\\' in the distant future. He fails in his initial missions to infiltrate South Park Elementary\\'s 4th Grade class, destroy the Trapper Keeper or Cartman himself, but with Stan Marsh and Kyle Broflovski\\'s aid, he is able to succeed in preventing his dismal future, and painfully fades from existence. South Park and all related titles, logos and characters are trademarks of Comedy Partners.\\\\\", \\\\\"score\\\\\": 0.5052006, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"\\\\\\\\\\\\\"South Park\\\\\\\\\\\\\" Clubhouses (TV Episode 1998) - IMDb\\\\\", \\\\\"url\\\\\": \\\\\"https://www.imdb.com/title/tt0705915/characters/nm0005295\\\\\", \\\\\"content\\\\\": \\\\\"\\\\\\\\\\\\\"South Park\\\\\\\\\\\\\" Clubhouses (TV Episode 1998) - Trey Parker as Stan Marsh, Eric Cartman, Phillip, Randy Marsh, Fat Abbot, Mr. Garrison, Mr. Mackey, 3rd Fat Abbot character, Roy, Teenage Boy #1, Clyde, Bill Cosby, Teenage Boy #2 - IMDb Awards & Events Trey Parker: Stan Marsh, Eric Cartman, Phillip, Randy Marsh, Fat Abbot, Mr. Garrison, Mr. Mackey, 3rd Fat Abbot character, Roy, Teenage Boy #1, Clyde, Bill Cosby, Teenage Boy #2 Mr. Garrison : Stan, are you paying attention? Stan : Yes, Mr. Garrison. Stan Marsh : Dare. Stan Marsh : What? Release Dates | Official Sites | Company Credits | Filming & Production | Technical Specs Photo & Video User Lists Related lists from IMDb users 2024 Watched TV Shows\\\\\", \\\\\"score\\\\\": 0.4604593, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Trapper Keeper (South Park) - Wikipedia\\\\\", \\\\\"url\\\\\": \\\\\"https://en.wikipedia.org/wiki/Trapper_Keeper_(South_Park)\\\\\", \\\\\"content\\\\\": \\\\\"\\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" is the twelfth episode of the fourth season of the animated television series South Park, and the 60th episode of the series overall. In the episode, a man from the future wants Cartman\\'s new Trapper Keeper, while Mr. Garrison\\'s kindergarten class holds an election for class president with confusing results. It is one of the many South Park episodes that parodies a current event.[1] The main plot of the episode involving the Trapper Keeper was written before the election,[1] but the subplot is a parody of the controversy surrounding the election\\'s outcome.[2] \\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" did not originally feature the election storyline, only a subplot about Ike attending his first day of kindergarten.[3] \\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" Full episode at South Park Studios\\\\\", \\\\\"score\\\\\": 0.3839421, \\\\\"raw_content\\\\\": null}]}\"}',\n", + "│ │ │ '{\"role\":\"assistant\",\"content\":\"Bill Cosby (BSM-471) first appears in the episode \\\\\"Trapper Keeper\\\\\" (Season 4, Episode 12) of South Park.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[]}',\n", "│ │ │ '{\"role\":\"user\",\"content\":\"What is the British-American kickboxer Andrew Tate\\'s kickboxing name?\",\"context\":null}'\n", - "│ │ ],\n", - "│ │ 'output': \"content: tool_calls: [ToolCall(call_id='526045a7-5f51-40fb-ba97-5ad29610e511', tool_name=<BuiltinTool.brave_search: 'brave_search'>, arguments={'query': 'Andrew Tate kickboxing name'})]\"\n", + "│ │ ],\n", + "│ │ 'output': \"content: tool_calls: [ToolCall(call_id='79276f65-3600-489d-ab41-d5a71dcaf075', tool_name=<BuiltinTool.brave_search: 'brave_search'>, arguments={'query': 'Andrew Tate kickboxing name'})]\"\n", "│ },\n", "│ {\n", - "│ │ 'input': '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"526045a7-5f51-40fb-ba97-5ad29610e511\",\"tool_name\":\"brave_search\",\"arguments\":{\"query\":\"Andrew Tate kickboxing name\"}}]}',\n", - "│ │ 'output': '{\"role\":\"ipython\",\"call_id\":\"526045a7-5f51-40fb-ba97-5ad29610e511\",\"tool_name\":\"brave_search\",\"content\":\"{\\\\\"query\\\\\": \\\\\"Andrew Tate kickboxing name\\\\\", \\\\\"top_k\\\\\": [{\\\\\"title\\\\\": \\\\\"Andrew Tate kickboxing record: How many championships ... - FirstSportz\\\\\", \\\\\"url\\\\\": \\\\\"https://firstsportz.com/mma-how-many-championships-does-andrew-tate-have/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate\\'s Kickboxing career. During his kickboxing career, he used the nickname \\\\\\\\\\\\\"King Cobra,\\\\\\\\\\\\\" which he currently uses as his Twitter name. Tate had an unorthodox style of movement inside the ring. He kept his hands down most of the time and relied on quick jabs and an overhand right to land significant strikes.\\\\\", \\\\\"score\\\\\": 0.9996244, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Andrew Tate: Kickboxing Record, Facts, Height, Weight, Age, Biography\\\\\", \\\\\"url\\\\\": \\\\\"https://www.lowkickmma.com/andrew-tate-kickboxing-record-facts-height-weight-age-biography/\\\\\", \\\\\"content\\\\\": \\\\\"Birth Name: Emory Andrew Tate III: Date of Birth: 1 December 1986: Place of Birth: Washington, D.C., U.S. ... In his professional kickboxing career, Andrew Tate won 32 of his fights by knockout.\\\\\", \\\\\"score\\\\\": 0.99909246, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Who is Andrew Tate? MMA, kickboxing record and controversies of fighter ...\\\\\", \\\\\"url\\\\\": \\\\\"https://www.sportingnews.com/us/kickboxing/news/andrew-tate-mma-kickboxing-record-controversies/u50waalc9cfz7krjg9wnyb7p\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate kickboxing record After launching his career as a 20-year-old in 2007, Tate built a formidable kickboxing record that included 76 wins across 85 fights in more than 13 years in the ring.\\\\\", \\\\\"score\\\\\": 0.9976586, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"About Andrew Tate: A Journey from Champion to Controversy\\\\\", \\\\\"url\\\\\": \\\\\"https://reachmorpheus.com/andrew-tate/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate\\'s kickboxing career, beginning in 2005, is a tale of determination and skill. He quickly made a name for himself in the sport, rising through the ranks with his unique fighting style and strategic approach, honed by his chess-playing background.\\\\\", \\\\\"score\\\\\": 0.99701905, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Andrew Tate Bio, Wiki, Net Worth, Age, Family, MMA Career - Next Biography\\\\\", \\\\\"url\\\\\": \\\\\"https://www.nextbiography.com/andrew-tate/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate Age. Andrew Tate is 36 years old as of 2023, born on December 1, 1986, in Washington, DC. By his mid-thirties, Andrew Tate has become an esteemed figure in the world of kickboxing, showcasing remarkable expertise and experience in the sport. Early Life of Andrew Tate. Andrew Tate was born on 01 December 1986 to an African-American\\\\\", \\\\\"score\\\\\": 0.99368566, \\\\\"raw_content\\\\\": null}]}\"}'\n", + "│ │ 'input': '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"79276f65-3600-489d-ab41-d5a71dcaf075\",\"tool_name\":\"brave_search\",\"arguments\":{\"query\":\"Andrew Tate kickboxing name\"}}]}',\n", + "│ │ 'output': '{\"role\":\"tool\",\"call_id\":\"79276f65-3600-489d-ab41-d5a71dcaf075\",\"tool_name\":\"brave_search\",\"content\":\"{\\\\\"query\\\\\": \\\\\"Andrew Tate kickboxing name\\\\\", \\\\\"top_k\\\\\": [{\\\\\"title\\\\\": \\\\\"Andrew Tate Age, Height, Weight, Family, Parents, Biography, Net Worth\\\\\", \\\\\"url\\\\\": \\\\\"https://biographywallah.com/andrew-tate-biography/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate Age, Height, Weight, Family, Parents, Biography, Net Worth \\\\\\\\u00bb Biography Wallah Andrew Tate Age, Height, Weight, Family, Parents, Biography, Net Worth Andrew Tate Biography NameAndrew TateReal nameEmory Andrew Tate IIIProfession \\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0Kickboxer, Commentator and BusinessmanDate of birth14 December 1986BirthplaceWashington D.C., United StatesAndrew Tate Age37 years old (as of 2024)NationalityBritish-AmericanZodiac SignSagittariusGenderMaleSchoolLocal School in Washington D.C., United StatesGirlfriend/SpouseNaghel GeorgianaSexual OrientationStraightNet worth$1000 Million Who is Andrew Tate? Andrew Tate is a British-American former professional kickboxing world champion businessman and media personality, who fought in the cruiserweight and super cruiserweight divisions. Andrew Tate Age Andrew Tate was born on 1 December 1986 and is 37 years old. Andrew Tate\\\\\\\\u2019s Net Worth What is the net worth of Andrew Tate? Where is Andrew Tate from? How old is Andrew Tate?\\\\\", \\\\\"score\\\\\": 0.80698997, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"The Life Of Andrew Tate (By Andrew Tate Himself ... - Sidekick Boxing\\\\\", \\\\\"url\\\\\": \\\\\"https://sidekickboxing.co.uk/the-life-of-andrew-king-cobra-tate/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate is a British-American former professional kickboxing world champion who fought in the cruiserweight and super cruiserweight divisions. Andrew Tate\\\\\\\\u2019s Kickboxing Career Andrew Tate in the Big Brother house Andrew Tate\\\\\\\\u2019s Kickboxing World Titles and his Sidekick boxing gloves Andrew Tate After Kickboxing Andrew Tate and his brother Tristan moved to Romania to set up their empire of businesses including trading in Bitcoin, Hustlers University, CobraTate.com, The Real World, and The War Room. From being a 4x kickboxing world champion to becoming the world\\\\\\\\u2019s most Googled man in the world with a private jet and over 33 cars, Andrew Tate\\\\\\\\u2019s life has been full of adventure.\\\\\", \\\\\"score\\\\\": 0.78194773, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Andrew Tate (\\\\\\\\\\\\\"King Cobra\\\\\\\\\\\\\") | MMA Fighter Page - Tapology\\\\\", \\\\\"url\\\\\": \\\\\"https://www.tapology.com/fightcenter/fighters/72139-andrew-tate\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate (\\\\\\\\\\\\\"King Cobra\\\\\\\\\\\\\") | MMA Fighter Page | Tapology Andrew \\\\\\\\\\\\\"King Cobra\\\\\\\\\\\\\" Tate Andrew Tate Name: Andrew Tate Height: 6\\'1\\\\\\\\\\\\\" (185cm) | Reach: Andrew Tate is ineligible for Tapology\\'s regional MMA rankings due to inactivity. Fighters must have at least one completed MMA bout in the past two years to be ranked. Andrew Tate MMA Fight Record Former top-ranked UFC fighter has called out Andrew Tate for having a paper title when it comes to combat... Andrew Tate \\\\\\\\u2022 All the biggest upcoming MMA & Boxing fights | UFC Fight Night | 02.01.2025, 12:00 PM ET | MMA Junkie: UFC Fight Night 249 video: Nine stoppages to open the year?! MMA Mania: Prochazka Vs. Hill: Odds, Full Fight Preview & Prediction\\\\\", \\\\\"score\\\\\": 0.6999322, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"About Andrew Tate: A Journey from Champion to Controversy\\\\\", \\\\\"url\\\\\": \\\\\"https://reachmorpheus.com/andrew-tate/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate\\'s kickboxing career, beginning in 2005, is a tale of determination and skill. He quickly made a name for himself in the sport, rising through the ranks with his unique fighting style and strategic approach, honed by his chess-playing background.\\\\\", \\\\\"score\\\\\": 0.6490677, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Andrew Tate\\'s Kickboxing Career & Biography - MMA Full Contact\\\\\", \\\\\"url\\\\\": \\\\\"https://www.mmafullcontact.com/andrew-tate-kickboxing/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate\\'s Kickboxing Career & Biography - MMA Full Contact Andrew Tate\\\\\\\\u2019s Kickboxing Career & Biography 2 Notable Opponents and Fights in Andrew Tate\\\\\\\\u2019s Kickboxing Career 4 Will Andrew Tate fight KSI? Notable Opponents and Fights in Andrew Tate\\\\\\\\u2019s Kickboxing Career Will Andrew Tate fight KSI? Similarly, Andrew Tate, known for his successful kickboxing career, has also shown interest in a potential fight with KSI. In conclusion, while there\\\\\\\\u2019s been plenty of interest and discussion about a potential boxing match between KSI and Andrew Tate, no official confirmation has been made as of now. With KSI\\\\\\\\u2019s upcoming match and Tate\\\\\\\\u2019s current personal circumstances, fans and followers of both personalities will have to wait for more updates on this potential fight.\\\\\", \\\\\"score\\\\\": 0.53050464, \\\\\"raw_content\\\\\": null}]}\"}'\n", "│ },\n", "│ {\n", "│ │ 'input': [\n", "│ │ │ '{\"role\":\"system\",\"content\":\"You are a helpful assistant. Use search tool to answer the questions. \"}',\n", "│ │ │ '{\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null}',\n", - "│ │ │ '{\"role\":\"assistant\",\"content\":\"Let me check the latest sports news.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[]}',\n", + "│ │ │ '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"8b7294ec-a83f-4798-ad8f-6bed662f08b6\",\"tool_name\":\"brave_search\",\"arguments\":{\"query\":\"NBA Western Conference Finals 2024 teams\"}}]}',\n", + "│ │ │ '{\"role\":\"tool\",\"call_id\":\"8b7294ec-a83f-4798-ad8f-6bed662f08b6\",\"tool_name\":\"brave_search\",\"content\":\"{\\\\\"query\\\\\": \\\\\"NBA Western Conference Finals 2024 teams\\\\\", \\\\\"top_k\\\\\": [{\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference Finals - Basketball-Reference.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.basketball-reference.com/playoffs/2024-nba-western-conference-finals-mavericks-vs-timberwolves.html\\\\\", \\\\\"content\\\\\": \\\\\"2024 NBA Western Conference Finals Mavericks vs. Timberwolves League Champion: Boston Celtics. Finals MVP: Jaylen Brown (20.8 / 5.4 / 5.0) 2024 Playoff Leaders: PTS: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (635) TRB: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (208) AST: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (178) WS: Derrick White (2.9) More playoffs info\\\\\", \\\\\"score\\\\\": 0.9310187, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates, schedule and more - Sportskeeda\\\\\", \\\\\"url\\\\\": \\\\\"https://www.sportskeeda.com/basketball/news-nba-western-conference-finals-2024-dates-schedule-and-more\\\\\", \\\\\"content\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates & Schedule The 2023-24 NBA Western Conference Finals will start on Wednesday, May 22. The Mavericks will face the team that wins in Game 7 between the\\\\\", \\\\\"score\\\\\": 0.8914433, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"2024 Playoffs: West Finals | Timberwolves (3) vs. Mavericks (5) - NBA.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.nba.com/playoffs/2024/west-final\\\\\", \\\\\"content\\\\\": \\\\\"The Dallas Mavericks and Minnesota Timberwolves have advanced to the 2024 Western Conference Finals during the NBA playoffs.\\\\\", \\\\\"score\\\\\": 0.8884594, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference playoff bracket - Basketnews.com\\\\\", \\\\\"url\\\\\": \\\\\"https://basketnews.com/news-204687-2024-nba-western-conference-playoff-bracket.html\\\\\", \\\\\"content\\\\\": \\\\\"In the 2024 NBA Western Conference playoffs, the Oklahoma City Thunder clinched the No. 1 seed. Every team from the Western Conference played their final game of the regular season, and two playoff pairs have been confirmed. The Los Angeles Lakers beat the New Orleans Pelicans, 110-106, in the Play-In Tournament to secure the 7th seed to set up a first-round matchup with the Denver Nuggets. Meanwhile, the Sacramento Kings will host the Golden State Warriors in the second Western Conference NBA Play-In Tournament game. The winners secure the No. 8 seed in the NBA playoffs for its conference. EuroLeague Play-In: Baskonia-Virtus game schedule announced\\\\\", \\\\\"score\\\\\": 0.8479807, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"NBA Finals 2024 - Celtics-Mavericks news, schedule, scores and ... - ESPN\\\\\", \\\\\"url\\\\\": \\\\\"https://www.espn.com/nba/story/_/id/39943302/nba-playoffs-2024-conference-finals-news-scores-highlights\\\\\", \\\\\"content\\\\\": \\\\\"The Boston Celtics are the 2024 NBA Champions. ... Western Conference. Final 2023-24 NBA regular-season standings. Which team left standing has the most trips to the NBA Finals? Here is a look at\\\\\", \\\\\"score\\\\\": 0.81979275, \\\\\"raw_content\\\\\": null}]}\"}',\n", + "│ │ │ '{\"role\":\"assistant\",\"content\":\"The teams that played in the NBA Western Conference Finals of 2024 were the Dallas Mavericks and the Minnesota Timberwolves.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[]}',\n", "│ │ │ '{\"role\":\"user\",\"content\":\"In which episode and season of South Park does Bill Cosby (BSM-471) first appear? Give me the number and title.\",\"context\":null}',\n", - "│ │ │ '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"19bd3554-e670-4856-89d0-c63f5b016245\",\"tool_name\":\"bravy_search\",\"arguments\":{\"query\":\"Bill Cosby South Park episode\"}}]}',\n", + "│ │ │ '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"fc0441bf-05ad-48d0-8034-4e19cb835904\",\"tool_name\":\"brave_search\",\"arguments\":{\"query\":\"Bill Cosby South Park episode\"}}]}',\n", + "│ │ │ '{\"role\":\"tool\",\"call_id\":\"fc0441bf-05ad-48d0-8034-4e19cb835904\",\"tool_name\":\"brave_search\",\"content\":\"{\\\\\"query\\\\\": \\\\\"Bill Cosby South Park episode\\\\\", \\\\\"top_k\\\\\": [{\\\\\"title\\\\\": \\\\\"Bill Cosby and Taylor Swift Duet - South Park Studios\\\\\", \\\\\"url\\\\\": \\\\\"https://www.southparkstudios.com/video-clips/90r7i1/south-park-bill-cosby-and-taylor-swift-duet\\\\\", \\\\\"content\\\\\": \\\\\"01:05 Bill Cosby is Here to See You South ParkS18 E10 ---------------------------------------------------- Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. 01:03 Bill Cosby and Taylor Swift Duet South ParkS18 E10 ------------------------------------------------------- The holiday special continues with Bill Cosby and Taylor Swift\\'s rendition of \\\\\\\\\\\\\"It\\'s Snowing Out There\\\\\\\\\\\\\". 01:31 #WeBelieveInYou South ParkS18 E10 -------------------------------------- With everyone watching, Kyle takes the opportunity to reach out to his brother. 01:47 Watch Your Microaggressions, Bro South ParkS19 E1 ------------------------------------------------------ Cartman\\'s plan to frame PC Principal backfires. South ParkS19 E1 -------------------------------------- After hearing that the PC people have targeted Kyle, Cartman vows to help.\\\\\", \\\\\"score\\\\\": 0.685971, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Bill Cosby is Here to See You - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/video-clips/wfot8s/south-park-bill-cosby-is-here-to-see-you\\\\\", \\\\\"content\\\\\": \\\\\"01:56 It\\'s Not About Music South ParkS18 E9 ------------------------------------------ At home, Randy sees the consequences of Lorde\\'s performance and calls the Record Producer to try and fix it. 01:24 Lorde\\'s Hologram South ParkS18 E9 -------------------------------------- The Record Producer reveals the truth about the music industry... South ParkS18 E9 --------------------------------------------- Randy catches Sharon with Tupac\\'s hologram. 01:37 I\\'ve Got Your Son, Lorde South ParkS18 E10 ----------------------------------------------- The Record Producer takes Stan and Kyle hostage. 01:05 Bill Cosby is Here to See You South ParkS18 E10 ---------------------------------------------------- Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. 01:21 Lorde Is My Dad South ParkS18 E10 -------------------------------------- After trying to confront Cartman Bra, Stan finally reveals the truth about his dad.\\\\\", \\\\\"score\\\\\": 0.6643884, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Bill Cosby (android) | South Park Character ... - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/wiki/Bill_Cosby_(android)\\\\\", \\\\\"content\\\\\": \\\\\"Bill Cosby (android) | South Park Character / Location / User talk etc | Official South Park Studios Wiki Sent back in time to destroy Eric Cartman\\'s Dawson\\'s Creek Trapper Keeper before it manifests into an omnipotent supercomputer that can destroy all humanity, \\\\\\\\\\\\\"Bill Cosby\\\\\\\\\\\\\" is really VSM471, an android or cyborg of some kind engineered by \\'hoomans\\' in the distant future. He fails in his initial missions to infiltrate South Park Elementary\\'s 4th Grade class, destroy the Trapper Keeper or Cartman himself, but with Stan Marsh and Kyle Broflovski\\'s aid, he is able to succeed in preventing his dismal future, and painfully fades from existence. South Park and all related titles, logos and characters are trademarks of Comedy Partners.\\\\\", \\\\\"score\\\\\": 0.5052006, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"\\\\\\\\\\\\\"South Park\\\\\\\\\\\\\" Clubhouses (TV Episode 1998) - IMDb\\\\\", \\\\\"url\\\\\": \\\\\"https://www.imdb.com/title/tt0705915/characters/nm0005295\\\\\", \\\\\"content\\\\\": \\\\\"\\\\\\\\\\\\\"South Park\\\\\\\\\\\\\" Clubhouses (TV Episode 1998) - Trey Parker as Stan Marsh, Eric Cartman, Phillip, Randy Marsh, Fat Abbot, Mr. Garrison, Mr. Mackey, 3rd Fat Abbot character, Roy, Teenage Boy #1, Clyde, Bill Cosby, Teenage Boy #2 - IMDb Awards & Events Trey Parker: Stan Marsh, Eric Cartman, Phillip, Randy Marsh, Fat Abbot, Mr. Garrison, Mr. Mackey, 3rd Fat Abbot character, Roy, Teenage Boy #1, Clyde, Bill Cosby, Teenage Boy #2 Mr. Garrison : Stan, are you paying attention? Stan : Yes, Mr. Garrison. Stan Marsh : Dare. Stan Marsh : What? Release Dates | Official Sites | Company Credits | Filming & Production | Technical Specs Photo & Video User Lists Related lists from IMDb users 2024 Watched TV Shows\\\\\", \\\\\"score\\\\\": 0.4604593, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Trapper Keeper (South Park) - Wikipedia\\\\\", \\\\\"url\\\\\": \\\\\"https://en.wikipedia.org/wiki/Trapper_Keeper_(South_Park)\\\\\", \\\\\"content\\\\\": \\\\\"\\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" is the twelfth episode of the fourth season of the animated television series South Park, and the 60th episode of the series overall. In the episode, a man from the future wants Cartman\\'s new Trapper Keeper, while Mr. Garrison\\'s kindergarten class holds an election for class president with confusing results. It is one of the many South Park episodes that parodies a current event.[1] The main plot of the episode involving the Trapper Keeper was written before the election,[1] but the subplot is a parody of the controversy surrounding the election\\'s outcome.[2] \\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" did not originally feature the election storyline, only a subplot about Ike attending his first day of kindergarten.[3] \\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" Full episode at South Park Studios\\\\\", \\\\\"score\\\\\": 0.3839421, \\\\\"raw_content\\\\\": null}]}\"}',\n", + "│ │ │ '{\"role\":\"assistant\",\"content\":\"Bill Cosby (BSM-471) first appears in the episode \\\\\"Trapper Keeper\\\\\" (Season 4, Episode 12) of South Park.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[]}',\n", "│ │ │ '{\"role\":\"user\",\"content\":\"What is the British-American kickboxer Andrew Tate\\'s kickboxing name?\",\"context\":null}',\n", - "│ │ │ '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"526045a7-5f51-40fb-ba97-5ad29610e511\",\"tool_name\":\"brave_search\",\"arguments\":{\"query\":\"Andrew Tate kickboxing name\"}}]}',\n", - "│ │ │ '{\"role\":\"ipython\",\"call_id\":\"526045a7-5f51-40fb-ba97-5ad29610e511\",\"tool_name\":\"brave_search\",\"content\":\"{\\\\\"query\\\\\": \\\\\"Andrew Tate kickboxing name\\\\\", \\\\\"top_k\\\\\": [{\\\\\"title\\\\\": \\\\\"Andrew Tate kickboxing record: How many championships ... - FirstSportz\\\\\", \\\\\"url\\\\\": \\\\\"https://firstsportz.com/mma-how-many-championships-does-andrew-tate-have/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate\\'s Kickboxing career. During his kickboxing career, he used the nickname \\\\\\\\\\\\\"King Cobra,\\\\\\\\\\\\\" which he currently uses as his Twitter name. Tate had an unorthodox style of movement inside the ring. He kept his hands down most of the time and relied on quick jabs and an overhand right to land significant strikes.\\\\\", \\\\\"score\\\\\": 0.9996244, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Andrew Tate: Kickboxing Record, Facts, Height, Weight, Age, Biography\\\\\", \\\\\"url\\\\\": \\\\\"https://www.lowkickmma.com/andrew-tate-kickboxing-record-facts-height-weight-age-biography/\\\\\", \\\\\"content\\\\\": \\\\\"Birth Name: Emory Andrew Tate III: Date of Birth: 1 December 1986: Place of Birth: Washington, D.C., U.S. ... In his professional kickboxing career, Andrew Tate won 32 of his fights by knockout.\\\\\", \\\\\"score\\\\\": 0.99909246, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Who is Andrew Tate? MMA, kickboxing record and controversies of fighter ...\\\\\", \\\\\"url\\\\\": \\\\\"https://www.sportingnews.com/us/kickboxing/news/andrew-tate-mma-kickboxing-record-controversies/u50waalc9cfz7krjg9wnyb7p\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate kickboxing record After launching his career as a 20-year-old in 2007, Tate built a formidable kickboxing record that included 76 wins across 85 fights in more than 13 years in the ring.\\\\\", \\\\\"score\\\\\": 0.9976586, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"About Andrew Tate: A Journey from Champion to Controversy\\\\\", \\\\\"url\\\\\": \\\\\"https://reachmorpheus.com/andrew-tate/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate\\'s kickboxing career, beginning in 2005, is a tale of determination and skill. He quickly made a name for himself in the sport, rising through the ranks with his unique fighting style and strategic approach, honed by his chess-playing background.\\\\\", \\\\\"score\\\\\": 0.99701905, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Andrew Tate Bio, Wiki, Net Worth, Age, Family, MMA Career - Next Biography\\\\\", \\\\\"url\\\\\": \\\\\"https://www.nextbiography.com/andrew-tate/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate Age. Andrew Tate is 36 years old as of 2023, born on December 1, 1986, in Washington, DC. By his mid-thirties, Andrew Tate has become an esteemed figure in the world of kickboxing, showcasing remarkable expertise and experience in the sport. Early Life of Andrew Tate. Andrew Tate was born on 01 December 1986 to an African-American\\\\\", \\\\\"score\\\\\": 0.99368566, \\\\\"raw_content\\\\\": null}]}\"}'\n", + "│ │ │ '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"79276f65-3600-489d-ab41-d5a71dcaf075\",\"tool_name\":\"brave_search\",\"arguments\":{\"query\":\"Andrew Tate kickboxing name\"}}]}',\n", + "│ │ │ '{\"role\":\"tool\",\"call_id\":\"79276f65-3600-489d-ab41-d5a71dcaf075\",\"tool_name\":\"brave_search\",\"content\":\"{\\\\\"query\\\\\": \\\\\"Andrew Tate kickboxing name\\\\\", \\\\\"top_k\\\\\": [{\\\\\"title\\\\\": \\\\\"Andrew Tate Age, Height, Weight, Family, Parents, Biography, Net Worth\\\\\", \\\\\"url\\\\\": \\\\\"https://biographywallah.com/andrew-tate-biography/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate Age, Height, Weight, Family, Parents, Biography, Net Worth \\\\\\\\u00bb Biography Wallah Andrew Tate Age, Height, Weight, Family, Parents, Biography, Net Worth Andrew Tate Biography NameAndrew TateReal nameEmory Andrew Tate IIIProfession \\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0Kickboxer, Commentator and BusinessmanDate of birth14 December 1986BirthplaceWashington D.C., United StatesAndrew Tate Age37 years old (as of 2024)NationalityBritish-AmericanZodiac SignSagittariusGenderMaleSchoolLocal School in Washington D.C., United StatesGirlfriend/SpouseNaghel GeorgianaSexual OrientationStraightNet worth$1000 Million Who is Andrew Tate? Andrew Tate is a British-American former professional kickboxing world champion businessman and media personality, who fought in the cruiserweight and super cruiserweight divisions. Andrew Tate Age Andrew Tate was born on 1 December 1986 and is 37 years old. Andrew Tate\\\\\\\\u2019s Net Worth What is the net worth of Andrew Tate? Where is Andrew Tate from? How old is Andrew Tate?\\\\\", \\\\\"score\\\\\": 0.80698997, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"The Life Of Andrew Tate (By Andrew Tate Himself ... - Sidekick Boxing\\\\\", \\\\\"url\\\\\": \\\\\"https://sidekickboxing.co.uk/the-life-of-andrew-king-cobra-tate/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate is a British-American former professional kickboxing world champion who fought in the cruiserweight and super cruiserweight divisions. Andrew Tate\\\\\\\\u2019s Kickboxing Career Andrew Tate in the Big Brother house Andrew Tate\\\\\\\\u2019s Kickboxing World Titles and his Sidekick boxing gloves Andrew Tate After Kickboxing Andrew Tate and his brother Tristan moved to Romania to set up their empire of businesses including trading in Bitcoin, Hustlers University, CobraTate.com, The Real World, and The War Room. From being a 4x kickboxing world champion to becoming the world\\\\\\\\u2019s most Googled man in the world with a private jet and over 33 cars, Andrew Tate\\\\\\\\u2019s life has been full of adventure.\\\\\", \\\\\"score\\\\\": 0.78194773, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Andrew Tate (\\\\\\\\\\\\\"King Cobra\\\\\\\\\\\\\") | MMA Fighter Page - Tapology\\\\\", \\\\\"url\\\\\": \\\\\"https://www.tapology.com/fightcenter/fighters/72139-andrew-tate\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate (\\\\\\\\\\\\\"King Cobra\\\\\\\\\\\\\") | MMA Fighter Page | Tapology Andrew \\\\\\\\\\\\\"King Cobra\\\\\\\\\\\\\" Tate Andrew Tate Name: Andrew Tate Height: 6\\'1\\\\\\\\\\\\\" (185cm) | Reach: Andrew Tate is ineligible for Tapology\\'s regional MMA rankings due to inactivity. Fighters must have at least one completed MMA bout in the past two years to be ranked. Andrew Tate MMA Fight Record Former top-ranked UFC fighter has called out Andrew Tate for having a paper title when it comes to combat... Andrew Tate \\\\\\\\u2022 All the biggest upcoming MMA & Boxing fights | UFC Fight Night | 02.01.2025, 12:00 PM ET | MMA Junkie: UFC Fight Night 249 video: Nine stoppages to open the year?! MMA Mania: Prochazka Vs. Hill: Odds, Full Fight Preview & Prediction\\\\\", \\\\\"score\\\\\": 0.6999322, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"About Andrew Tate: A Journey from Champion to Controversy\\\\\", \\\\\"url\\\\\": \\\\\"https://reachmorpheus.com/andrew-tate/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate\\'s kickboxing career, beginning in 2005, is a tale of determination and skill. He quickly made a name for himself in the sport, rising through the ranks with his unique fighting style and strategic approach, honed by his chess-playing background.\\\\\", \\\\\"score\\\\\": 0.6490677, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Andrew Tate\\'s Kickboxing Career & Biography - MMA Full Contact\\\\\", \\\\\"url\\\\\": \\\\\"https://www.mmafullcontact.com/andrew-tate-kickboxing/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate\\'s Kickboxing Career & Biography - MMA Full Contact Andrew Tate\\\\\\\\u2019s Kickboxing Career & Biography 2 Notable Opponents and Fights in Andrew Tate\\\\\\\\u2019s Kickboxing Career 4 Will Andrew Tate fight KSI? Notable Opponents and Fights in Andrew Tate\\\\\\\\u2019s Kickboxing Career Will Andrew Tate fight KSI? Similarly, Andrew Tate, known for his successful kickboxing career, has also shown interest in a potential fight with KSI. In conclusion, while there\\\\\\\\u2019s been plenty of interest and discussion about a potential boxing match between KSI and Andrew Tate, no official confirmation has been made as of now. With KSI\\\\\\\\u2019s upcoming match and Tate\\\\\\\\u2019s current personal circumstances, fans and followers of both personalities will have to wait for more updates on this potential fight.\\\\\", \\\\\"score\\\\\": 0.53050464, \\\\\"raw_content\\\\\": null}]}\"}'\n", "│ │ ],\n", "│ │ 'output': 'content: Andrew Tate\\'s kickboxing name is \"King Cobra.\" tool_calls: []'\n", "│ }\n", @@ -2216,42 +3436,82 @@ "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"system\",\"content\":\"You are a helpful assistant. Use search tool to answer the questions. \"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\n", "\u001b[2;32m│ │ \u001b[0m\u001b[1m]\u001b[0m,\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'output'\u001b[0m: \u001b[32m'content: Let me check the latest sports news. tool_calls: \u001b[0m\u001b[32m[\u001b[0m\u001b[32m]\u001b[0m\u001b[32m'\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[1m}\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[1m{\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'input'\u001b[0m: \u001b[1m[\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"system\",\"content\":\"You are a helpful assistant. Use search tool to answer the questions. \"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"Let me check the latest sports news.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[32m'output'\u001b[0m: \u001b[32m\"content: tool_calls: \u001b[0m\u001b[32m[\u001b[0m\u001b[32mToolCall\u001b[0m\u001b[32m(\u001b[0m\u001b[32mcall_id\u001b[0m\u001b[32m='8b7294ec-a83f-4798-ad8f-6bed662f08b6', \u001b[0m\u001b[32mtool_name\u001b[0m\u001b[32m=\u001b[0m\u001b[32m<\u001b[0m\u001b[32mBuiltinTool.brave_search:\u001b[0m\u001b[32m 'brave_search'>, \u001b[0m\u001b[32marguments\u001b[0m\u001b[32m=\u001b[0m\u001b[32m{\u001b[0m\u001b[32m'query': 'NBA Western Conference Finals 2024 teams'\u001b[0m\u001b[32m}\u001b[0m\u001b[32m)\u001b[0m\u001b[32m]\u001b[0m\u001b[32m\"\u001b[0m\n", + "\u001b[2;32m│ \u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ \u001b[0m\u001b[1;39m{\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[32m'input'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"call_id\":\"8b7294ec-a83f-4798-ad8f-6bed662f08b6\",\"tool_name\":\"brave_search\",\"arguments\":\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"query\":\"NBA Western Conference Finals 2024 teams\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[32m'output'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"tool\",\"call_id\":\"8b7294ec-a83f-4798-ad8f-6bed662f08b6\",\"tool_name\":\"brave_search\",\"content\":\"\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"query\\\\\": \\\\\"NBA Western Conference Finals 2024 teams\\\\\", \\\\\"top_k\\\\\": \u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference Finals - Basketball-Reference.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.basketball-reference.com/playoffs/2024-nba-western-conference-finals-mavericks-vs-timberwolves.html\\\\\", \\\\\"content\\\\\": \\\\\"2024 NBA Western Conference Finals Mavericks vs. Timberwolves League Champion: Boston Celtics. Finals MVP: Jaylen Brown \u001b[0m\u001b[32m(\u001b[0m\u001b[32m20.8 / 5.4 / 5.0\u001b[0m\u001b[32m)\u001b[0m\u001b[32m 2024 Playoff Leaders: PTS: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m635\u001b[0m\u001b[32m)\u001b[0m\u001b[32m TRB: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m208\u001b[0m\u001b[32m)\u001b[0m\u001b[32m AST: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m178\u001b[0m\u001b[32m)\u001b[0m\u001b[32m WS: Derrick White \u001b[0m\u001b[32m(\u001b[0m\u001b[32m2.9\u001b[0m\u001b[32m)\u001b[0m\u001b[32m More playoffs info\\\\\", \\\\\"score\\\\\": 0.9310187, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates, schedule and more - Sportskeeda\\\\\", \\\\\"url\\\\\": \\\\\"https://www.sportskeeda.com/basketball/news-nba-western-conference-finals-2024-dates-schedule-and-more\\\\\", \\\\\"content\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates & Schedule The 2023-24 NBA Western Conference Finals will start on Wednesday, May 22. The Mavericks will face the team that wins in Game 7 between the\\\\\", \\\\\"score\\\\\": 0.8914433, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"2024 Playoffs: West Finals | Timberwolves \u001b[0m\u001b[32m(\u001b[0m\u001b[32m3\u001b[0m\u001b[32m)\u001b[0m\u001b[32m vs. Mavericks \u001b[0m\u001b[32m(\u001b[0m\u001b[32m5\u001b[0m\u001b[32m)\u001b[0m\u001b[32m - NBA.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.nba.com/playoffs/2024/west-final\\\\\", \\\\\"content\\\\\": \\\\\"The Dallas Mavericks and Minnesota Timberwolves have advanced to the 2024 Western Conference Finals during the NBA playoffs.\\\\\", \\\\\"score\\\\\": 0.8884594, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference playoff bracket - Basketnews.com\\\\\", \\\\\"url\\\\\": \\\\\"https://basketnews.com/news-204687-2024-nba-western-conference-playoff-bracket.html\\\\\", \\\\\"content\\\\\": \\\\\"In the 2024 NBA Western Conference playoffs, the Oklahoma City Thunder clinched the No. 1 seed. Every team from the Western Conference played their final game of the regular season, and two playoff pairs have been confirmed. The Los Angeles Lakers beat the New Orleans Pelicans, 110-106, in the Play-In Tournament to secure the 7th seed to set up a first-round matchup with the Denver Nuggets. Meanwhile, the Sacramento Kings will host the Golden State Warriors in the second Western Conference NBA Play-In Tournament game. The winners secure the No. 8 seed in the NBA playoffs for its conference. EuroLeague Play-In: Baskonia-Virtus game schedule announced\\\\\", \\\\\"score\\\\\": 0.8479807, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"NBA Finals 2024 - Celtics-Mavericks news, schedule, scores and ... - ESPN\\\\\", \\\\\"url\\\\\": \\\\\"https://www.espn.com/nba/story/_/id/39943302/nba-playoffs-2024-conference-finals-news-scores-highlights\\\\\", \\\\\"content\\\\\": \\\\\"The Boston Celtics are the 2024 NBA Champions. ... Western Conference. Final 2023-24 NBA regular-season standings. Which team left standing has the most trips to the NBA Finals? Here is a look at\\\\\", \\\\\"score\\\\\": 0.81979275, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\n", + "\u001b[2;32m│ \u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ \u001b[0m\u001b[1;39m{\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[32m'input'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;39m[\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"system\",\"content\":\"You are a helpful assistant. Use search tool to answer the questions. \"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"call_id\":\"8b7294ec-a83f-4798-ad8f-6bed662f08b6\",\"tool_name\":\"brave_search\",\"arguments\":\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"query\":\"NBA Western Conference Finals 2024 teams\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"tool\",\"call_id\":\"8b7294ec-a83f-4798-ad8f-6bed662f08b6\",\"tool_name\":\"brave_search\",\"content\":\"\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"query\\\\\": \\\\\"NBA Western Conference Finals 2024 teams\\\\\", \\\\\"top_k\\\\\": \u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference Finals - Basketball-Reference.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.basketball-reference.com/playoffs/2024-nba-western-conference-finals-mavericks-vs-timberwolves.html\\\\\", \\\\\"content\\\\\": \\\\\"2024 NBA Western Conference Finals Mavericks vs. Timberwolves League Champion: Boston Celtics. Finals MVP: Jaylen Brown \u001b[0m\u001b[32m(\u001b[0m\u001b[32m20.8 / 5.4 / 5.0\u001b[0m\u001b[32m)\u001b[0m\u001b[32m 2024 Playoff Leaders: PTS: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m635\u001b[0m\u001b[32m)\u001b[0m\u001b[32m TRB: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m208\u001b[0m\u001b[32m)\u001b[0m\u001b[32m AST: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m178\u001b[0m\u001b[32m)\u001b[0m\u001b[32m WS: Derrick White \u001b[0m\u001b[32m(\u001b[0m\u001b[32m2.9\u001b[0m\u001b[32m)\u001b[0m\u001b[32m More playoffs info\\\\\", \\\\\"score\\\\\": 0.9310187, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates, schedule and more - Sportskeeda\\\\\", \\\\\"url\\\\\": \\\\\"https://www.sportskeeda.com/basketball/news-nba-western-conference-finals-2024-dates-schedule-and-more\\\\\", \\\\\"content\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates & Schedule The 2023-24 NBA Western Conference Finals will start on Wednesday, May 22. The Mavericks will face the team that wins in Game 7 between the\\\\\", \\\\\"score\\\\\": 0.8914433, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"2024 Playoffs: West Finals | Timberwolves \u001b[0m\u001b[32m(\u001b[0m\u001b[32m3\u001b[0m\u001b[32m)\u001b[0m\u001b[32m vs. Mavericks \u001b[0m\u001b[32m(\u001b[0m\u001b[32m5\u001b[0m\u001b[32m)\u001b[0m\u001b[32m - NBA.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.nba.com/playoffs/2024/west-final\\\\\", \\\\\"content\\\\\": \\\\\"The Dallas Mavericks and Minnesota Timberwolves have advanced to the 2024 Western Conference Finals during the NBA playoffs.\\\\\", \\\\\"score\\\\\": 0.8884594, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference playoff bracket - Basketnews.com\\\\\", \\\\\"url\\\\\": \\\\\"https://basketnews.com/news-204687-2024-nba-western-conference-playoff-bracket.html\\\\\", \\\\\"content\\\\\": \\\\\"In the 2024 NBA Western Conference playoffs, the Oklahoma City Thunder clinched the No. 1 seed. Every team from the Western Conference played their final game of the regular season, and two playoff pairs have been confirmed. The Los Angeles Lakers beat the New Orleans Pelicans, 110-106, in the Play-In Tournament to secure the 7th seed to set up a first-round matchup with the Denver Nuggets. Meanwhile, the Sacramento Kings will host the Golden State Warriors in the second Western Conference NBA Play-In Tournament game. The winners secure the No. 8 seed in the NBA playoffs for its conference. EuroLeague Play-In: Baskonia-Virtus game schedule announced\\\\\", \\\\\"score\\\\\": 0.8479807, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"NBA Finals 2024 - Celtics-Mavericks news, schedule, scores and ... - ESPN\\\\\", \\\\\"url\\\\\": \\\\\"https://www.espn.com/nba/story/_/id/39943302/nba-playoffs-2024-conference-finals-news-scores-highlights\\\\\", \\\\\"content\\\\\": \\\\\"The Boston Celtics are the 2024 NBA Champions. ... Western Conference. Final 2023-24 NBA regular-season standings. Which team left standing has the most trips to the NBA Finals? Here is a look at\\\\\", \\\\\"score\\\\\": 0.81979275, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[1;39m]\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[32m'output'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'content: The teams that played in the NBA Western Conference Finals of 2024 were the Dallas Mavericks and the Minnesota Timberwolves. tool_calls: \u001b[0m\u001b[32m[\u001b[0m\u001b[32m]\u001b[0m\u001b[32m'\u001b[0m\n", + "\u001b[2;32m│ \u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ \u001b[0m\u001b[1;39m{\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[32m'input'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;39m[\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"system\",\"content\":\"You are a helpful assistant. Use search tool to answer the questions. \"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"call_id\":\"8b7294ec-a83f-4798-ad8f-6bed662f08b6\",\"tool_name\":\"brave_search\",\"arguments\":\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"query\":\"NBA Western Conference Finals 2024 teams\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"tool\",\"call_id\":\"8b7294ec-a83f-4798-ad8f-6bed662f08b6\",\"tool_name\":\"brave_search\",\"content\":\"\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"query\\\\\": \\\\\"NBA Western Conference Finals 2024 teams\\\\\", \\\\\"top_k\\\\\": \u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference Finals - Basketball-Reference.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.basketball-reference.com/playoffs/2024-nba-western-conference-finals-mavericks-vs-timberwolves.html\\\\\", \\\\\"content\\\\\": \\\\\"2024 NBA Western Conference Finals Mavericks vs. Timberwolves League Champion: Boston Celtics. Finals MVP: Jaylen Brown \u001b[0m\u001b[32m(\u001b[0m\u001b[32m20.8 / 5.4 / 5.0\u001b[0m\u001b[32m)\u001b[0m\u001b[32m 2024 Playoff Leaders: PTS: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m635\u001b[0m\u001b[32m)\u001b[0m\u001b[32m TRB: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m208\u001b[0m\u001b[32m)\u001b[0m\u001b[32m AST: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m178\u001b[0m\u001b[32m)\u001b[0m\u001b[32m WS: Derrick White \u001b[0m\u001b[32m(\u001b[0m\u001b[32m2.9\u001b[0m\u001b[32m)\u001b[0m\u001b[32m More playoffs info\\\\\", \\\\\"score\\\\\": 0.9310187, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates, schedule and more - Sportskeeda\\\\\", \\\\\"url\\\\\": \\\\\"https://www.sportskeeda.com/basketball/news-nba-western-conference-finals-2024-dates-schedule-and-more\\\\\", \\\\\"content\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates & Schedule The 2023-24 NBA Western Conference Finals will start on Wednesday, May 22. The Mavericks will face the team that wins in Game 7 between the\\\\\", \\\\\"score\\\\\": 0.8914433, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"2024 Playoffs: West Finals | Timberwolves \u001b[0m\u001b[32m(\u001b[0m\u001b[32m3\u001b[0m\u001b[32m)\u001b[0m\u001b[32m vs. Mavericks \u001b[0m\u001b[32m(\u001b[0m\u001b[32m5\u001b[0m\u001b[32m)\u001b[0m\u001b[32m - NBA.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.nba.com/playoffs/2024/west-final\\\\\", \\\\\"content\\\\\": \\\\\"The Dallas Mavericks and Minnesota Timberwolves have advanced to the 2024 Western Conference Finals during the NBA playoffs.\\\\\", \\\\\"score\\\\\": 0.8884594, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference playoff bracket - Basketnews.com\\\\\", \\\\\"url\\\\\": \\\\\"https://basketnews.com/news-204687-2024-nba-western-conference-playoff-bracket.html\\\\\", \\\\\"content\\\\\": \\\\\"In the 2024 NBA Western Conference playoffs, the Oklahoma City Thunder clinched the No. 1 seed. Every team from the Western Conference played their final game of the regular season, and two playoff pairs have been confirmed. The Los Angeles Lakers beat the New Orleans Pelicans, 110-106, in the Play-In Tournament to secure the 7th seed to set up a first-round matchup with the Denver Nuggets. Meanwhile, the Sacramento Kings will host the Golden State Warriors in the second Western Conference NBA Play-In Tournament game. The winners secure the No. 8 seed in the NBA playoffs for its conference. EuroLeague Play-In: Baskonia-Virtus game schedule announced\\\\\", \\\\\"score\\\\\": 0.8479807, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"NBA Finals 2024 - Celtics-Mavericks news, schedule, scores and ... - ESPN\\\\\", \\\\\"url\\\\\": \\\\\"https://www.espn.com/nba/story/_/id/39943302/nba-playoffs-2024-conference-finals-news-scores-highlights\\\\\", \\\\\"content\\\\\": \\\\\"The Boston Celtics are the 2024 NBA Champions. ... Western Conference. Final 2023-24 NBA regular-season standings. Which team left standing has the most trips to the NBA Finals? Here is a look at\\\\\", \\\\\"score\\\\\": 0.81979275, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"The teams that played in the NBA Western Conference Finals of 2024 were the Dallas Mavericks and the Minnesota Timberwolves.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"user\",\"content\":\"In which episode and season of South Park does Bill Cosby \u001b[0m\u001b[32m(\u001b[0m\u001b[32mBSM-471\u001b[0m\u001b[32m)\u001b[0m\u001b[32m first appear? Give me the number and title.\",\"context\":null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[1m]\u001b[0m,\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'output'\u001b[0m: \u001b[32m\"content: tool_calls: \u001b[0m\u001b[32m[\u001b[0m\u001b[32mToolCall\u001b[0m\u001b[32m(\u001b[0m\u001b[32mcall_id\u001b[0m\u001b[32m='19bd3554-e670-4856-89d0-c63f5b016245', \u001b[0m\u001b[32mtool_name\u001b[0m\u001b[32m='bravy_search', \u001b[0m\u001b[32marguments\u001b[0m\u001b[32m=\u001b[0m\u001b[32m{\u001b[0m\u001b[32m'query': 'Bill Cosby South Park episode'\u001b[0m\u001b[32m}\u001b[0m\u001b[32m)\u001b[0m\u001b[32m]\u001b[0m\u001b[32m\"\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[1m}\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[1m{\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'input'\u001b[0m: \u001b[1m[\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"system\",\"content\":\"You are a helpful assistant. Use search tool to answer the questions. \"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"Let me check the latest sports news.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"user\",\"content\":\"In which episode and season of South Park does Bill Cosby \u001b[0m\u001b[32m(\u001b[0m\u001b[32mBSM-471\u001b[0m\u001b[32m)\u001b[0m\u001b[32m first appear? Give me the number and title.\",\"context\":null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"call_id\":\"19bd3554-e670-4856-89d0-c63f5b016245\",\"tool_name\":\"bravy_search\",\"arguments\":\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"query\":\"Bill Cosby South Park episode\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[1;39m]\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[32m'output'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m\"content: tool_calls: \u001b[0m\u001b[32m[\u001b[0m\u001b[32mToolCall\u001b[0m\u001b[32m(\u001b[0m\u001b[32mcall_id\u001b[0m\u001b[32m='fc0441bf-05ad-48d0-8034-4e19cb835904', \u001b[0m\u001b[32mtool_name\u001b[0m\u001b[32m=, \u001b[0m\u001b[32marguments\u001b[0m\u001b[32m=\u001b[0m\u001b[32m{\u001b[0m\u001b[32m'query': 'Bill Cosby South Park episode'\u001b[0m\u001b[32m}\u001b[0m\u001b[32m)\u001b[0m\u001b[32m]\u001b[0m\u001b[32m\"\u001b[0m\n", + "\u001b[2;32m│ \u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ \u001b[0m\u001b[1;39m{\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[32m'input'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"call_id\":\"fc0441bf-05ad-48d0-8034-4e19cb835904\",\"tool_name\":\"brave_search\",\"arguments\":\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"query\":\"Bill Cosby South Park episode\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[32m'output'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"tool\",\"call_id\":\"fc0441bf-05ad-48d0-8034-4e19cb835904\",\"tool_name\":\"brave_search\",\"content\":\"\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"query\\\\\": \\\\\"Bill Cosby South Park episode\\\\\", \\\\\"top_k\\\\\": \u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Bill Cosby and Taylor Swift Duet - South Park Studios\\\\\", \\\\\"url\\\\\": \\\\\"https://www.southparkstudios.com/video-clips/90r7i1/south-park-bill-cosby-and-taylor-swift-duet\\\\\", \\\\\"content\\\\\": \\\\\"01:05 Bill Cosby is Here to See You South ParkS18 E10 ---------------------------------------------------- Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. 01:03 Bill Cosby and Taylor Swift Duet South ParkS18 E10 ------------------------------------------------------- The holiday special continues with Bill Cosby and Taylor Swift\\'s rendition of \\\\\\\\\\\\\"It\\'s Snowing Out There\\\\\\\\\\\\\". 01:31 #WeBelieveInYou South ParkS18 E10 -------------------------------------- With everyone watching, Kyle takes the opportunity to reach out to his brother. 01:47 Watch Your Microaggressions, Bro South ParkS19 E1 ------------------------------------------------------ Cartman\\'s plan to frame PC Principal backfires. South ParkS19 E1 -------------------------------------- After hearing that the PC people have targeted Kyle, Cartman vows to help.\\\\\", \\\\\"score\\\\\": 0.685971, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Bill Cosby is Here to See You - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/video-clips/wfot8s/south-park-bill-cosby-is-here-to-see-you\\\\\", \\\\\"content\\\\\": \\\\\"01:56 It\\'s Not About Music South ParkS18 E9 ------------------------------------------ At home, Randy sees the consequences of Lorde\\'s performance and calls the Record Producer to try and fix it. 01:24 Lorde\\'s Hologram South ParkS18 E9 -------------------------------------- The Record Producer reveals the truth about the music industry... South ParkS18 E9 --------------------------------------------- Randy catches Sharon with Tupac\\'s hologram. 01:37 I\\'ve Got Your Son, Lorde South ParkS18 E10 ----------------------------------------------- The Record Producer takes Stan and Kyle hostage. 01:05 Bill Cosby is Here to See You South ParkS18 E10 ---------------------------------------------------- Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. 01:21 Lorde Is My Dad South ParkS18 E10 -------------------------------------- After trying to confront Cartman Bra, Stan finally reveals the truth about his dad.\\\\\", \\\\\"score\\\\\": 0.6643884, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Bill Cosby \u001b[0m\u001b[32m(\u001b[0m\u001b[32mandroid\u001b[0m\u001b[32m)\u001b[0m\u001b[32m | South Park Character ... - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/wiki/Bill_Cosby_\u001b[0m\u001b[32m(\u001b[0m\u001b[32mandroid\u001b[0m\u001b[32m)\u001b[0m\u001b[32m\\\\\", \\\\\"content\\\\\": \\\\\"Bill Cosby \u001b[0m\u001b[32m(\u001b[0m\u001b[32mandroid\u001b[0m\u001b[32m)\u001b[0m\u001b[32m | South Park Character / Location / User talk etc | Official South Park Studios Wiki Sent back in time to destroy Eric Cartman\\'s Dawson\\'s Creek Trapper Keeper before it manifests into an omnipotent supercomputer that can destroy all humanity, \\\\\\\\\\\\\"Bill Cosby\\\\\\\\\\\\\" is really VSM471, an android or cyborg of some kind engineered by \\'hoomans\\' in the distant future. He fails in his initial missions to infiltrate South Park Elementary\\'s 4th Grade class, destroy the Trapper Keeper or Cartman himself, but with Stan Marsh and Kyle Broflovski\\'s aid, he is able to succeed in preventing his dismal future, and painfully fades from existence. South Park and all related titles, logos and characters are trademarks of Comedy Partners.\\\\\", \\\\\"score\\\\\": 0.5052006, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"\\\\\\\\\\\\\"South Park\\\\\\\\\\\\\" Clubhouses \u001b[0m\u001b[32m(\u001b[0m\u001b[32mTV Episode 1998\u001b[0m\u001b[32m)\u001b[0m\u001b[32m - IMDb\\\\\", \\\\\"url\\\\\": \\\\\"https://www.imdb.com/title/tt0705915/characters/nm0005295\\\\\", \\\\\"content\\\\\": \\\\\"\\\\\\\\\\\\\"South Park\\\\\\\\\\\\\" Clubhouses \u001b[0m\u001b[32m(\u001b[0m\u001b[32mTV Episode 1998\u001b[0m\u001b[32m)\u001b[0m\u001b[32m - Trey Parker as Stan Marsh, Eric Cartman, Phillip, Randy Marsh, Fat Abbot, Mr. Garrison, Mr. Mackey, 3rd Fat Abbot character, Roy, Teenage Boy #1, Clyde, Bill Cosby, Teenage Boy #2 - IMDb Awards & Events Trey Parker: Stan Marsh, Eric Cartman, Phillip, Randy Marsh, Fat Abbot, Mr. Garrison, Mr. Mackey, 3rd Fat Abbot character, Roy, Teenage Boy #1, Clyde, Bill Cosby, Teenage Boy #2 Mr. Garrison : Stan, are you paying attention? Stan : Yes, Mr. Garrison. Stan Marsh : Dare. Stan Marsh : What? Release Dates | Official Sites | Company Credits | Filming & Production | Technical Specs Photo & Video User Lists Related lists from IMDb users 2024 Watched TV Shows\\\\\", \\\\\"score\\\\\": 0.4604593, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Trapper Keeper \u001b[0m\u001b[32m(\u001b[0m\u001b[32mSouth Park\u001b[0m\u001b[32m)\u001b[0m\u001b[32m - Wikipedia\\\\\", \\\\\"url\\\\\": \\\\\"https://en.wikipedia.org/wiki/Trapper_Keeper_\u001b[0m\u001b[32m(\u001b[0m\u001b[32mSouth_Park\u001b[0m\u001b[32m)\u001b[0m\u001b[32m\\\\\", \\\\\"content\\\\\": \\\\\"\\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" is the twelfth episode of the fourth season of the animated television series South Park, and the 60th episode of the series overall. In the episode, a man from the future wants Cartman\\'s new Trapper Keeper, while Mr. Garrison\\'s kindergarten class holds an election for class president with confusing results. It is one of the many South Park episodes that parodies a current event.\u001b[0m\u001b[32m[\u001b[0m\u001b[32m1\u001b[0m\u001b[32m]\u001b[0m\u001b[32m The main plot of the episode involving the Trapper Keeper was written before the election,\u001b[0m\u001b[32m[\u001b[0m\u001b[32m1\u001b[0m\u001b[32m]\u001b[0m\u001b[32m but the subplot is a parody of the controversy surrounding the election\\'s outcome.\u001b[0m\u001b[32m[\u001b[0m\u001b[32m2\u001b[0m\u001b[32m]\u001b[0m\u001b[32m \\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" did not originally feature the election storyline, only a subplot about Ike attending his first day of kindergarten.\u001b[0m\u001b[32m[\u001b[0m\u001b[32m3\u001b[0m\u001b[32m]\u001b[0m\u001b[32m \\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" Full episode at South Park Studios\\\\\", \\\\\"score\\\\\": 0.3839421, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\n", + "\u001b[2;32m│ \u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ \u001b[0m\u001b[1;39m{\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[32m'input'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;39m[\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"system\",\"content\":\"You are a helpful assistant. Use search tool to answer the questions. \"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"call_id\":\"8b7294ec-a83f-4798-ad8f-6bed662f08b6\",\"tool_name\":\"brave_search\",\"arguments\":\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"query\":\"NBA Western Conference Finals 2024 teams\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"tool\",\"call_id\":\"8b7294ec-a83f-4798-ad8f-6bed662f08b6\",\"tool_name\":\"brave_search\",\"content\":\"\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"query\\\\\": \\\\\"NBA Western Conference Finals 2024 teams\\\\\", \\\\\"top_k\\\\\": \u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference Finals - Basketball-Reference.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.basketball-reference.com/playoffs/2024-nba-western-conference-finals-mavericks-vs-timberwolves.html\\\\\", \\\\\"content\\\\\": \\\\\"2024 NBA Western Conference Finals Mavericks vs. Timberwolves League Champion: Boston Celtics. Finals MVP: Jaylen Brown \u001b[0m\u001b[32m(\u001b[0m\u001b[32m20.8 / 5.4 / 5.0\u001b[0m\u001b[32m)\u001b[0m\u001b[32m 2024 Playoff Leaders: PTS: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m635\u001b[0m\u001b[32m)\u001b[0m\u001b[32m TRB: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m208\u001b[0m\u001b[32m)\u001b[0m\u001b[32m AST: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m178\u001b[0m\u001b[32m)\u001b[0m\u001b[32m WS: Derrick White \u001b[0m\u001b[32m(\u001b[0m\u001b[32m2.9\u001b[0m\u001b[32m)\u001b[0m\u001b[32m More playoffs info\\\\\", \\\\\"score\\\\\": 0.9310187, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates, schedule and more - Sportskeeda\\\\\", \\\\\"url\\\\\": \\\\\"https://www.sportskeeda.com/basketball/news-nba-western-conference-finals-2024-dates-schedule-and-more\\\\\", \\\\\"content\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates & Schedule The 2023-24 NBA Western Conference Finals will start on Wednesday, May 22. The Mavericks will face the team that wins in Game 7 between the\\\\\", \\\\\"score\\\\\": 0.8914433, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"2024 Playoffs: West Finals | Timberwolves \u001b[0m\u001b[32m(\u001b[0m\u001b[32m3\u001b[0m\u001b[32m)\u001b[0m\u001b[32m vs. Mavericks \u001b[0m\u001b[32m(\u001b[0m\u001b[32m5\u001b[0m\u001b[32m)\u001b[0m\u001b[32m - NBA.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.nba.com/playoffs/2024/west-final\\\\\", \\\\\"content\\\\\": \\\\\"The Dallas Mavericks and Minnesota Timberwolves have advanced to the 2024 Western Conference Finals during the NBA playoffs.\\\\\", \\\\\"score\\\\\": 0.8884594, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference playoff bracket - Basketnews.com\\\\\", \\\\\"url\\\\\": \\\\\"https://basketnews.com/news-204687-2024-nba-western-conference-playoff-bracket.html\\\\\", \\\\\"content\\\\\": \\\\\"In the 2024 NBA Western Conference playoffs, the Oklahoma City Thunder clinched the No. 1 seed. Every team from the Western Conference played their final game of the regular season, and two playoff pairs have been confirmed. The Los Angeles Lakers beat the New Orleans Pelicans, 110-106, in the Play-In Tournament to secure the 7th seed to set up a first-round matchup with the Denver Nuggets. Meanwhile, the Sacramento Kings will host the Golden State Warriors in the second Western Conference NBA Play-In Tournament game. The winners secure the No. 8 seed in the NBA playoffs for its conference. EuroLeague Play-In: Baskonia-Virtus game schedule announced\\\\\", \\\\\"score\\\\\": 0.8479807, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"NBA Finals 2024 - Celtics-Mavericks news, schedule, scores and ... - ESPN\\\\\", \\\\\"url\\\\\": \\\\\"https://www.espn.com/nba/story/_/id/39943302/nba-playoffs-2024-conference-finals-news-scores-highlights\\\\\", \\\\\"content\\\\\": \\\\\"The Boston Celtics are the 2024 NBA Champions. ... Western Conference. Final 2023-24 NBA regular-season standings. Which team left standing has the most trips to the NBA Finals? Here is a look at\\\\\", \\\\\"score\\\\\": 0.81979275, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"The teams that played in the NBA Western Conference Finals of 2024 were the Dallas Mavericks and the Minnesota Timberwolves.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"user\",\"content\":\"In which episode and season of South Park does Bill Cosby \u001b[0m\u001b[32m(\u001b[0m\u001b[32mBSM-471\u001b[0m\u001b[32m)\u001b[0m\u001b[32m first appear? Give me the number and title.\",\"context\":null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"call_id\":\"fc0441bf-05ad-48d0-8034-4e19cb835904\",\"tool_name\":\"brave_search\",\"arguments\":\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"query\":\"Bill Cosby South Park episode\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"tool\",\"call_id\":\"fc0441bf-05ad-48d0-8034-4e19cb835904\",\"tool_name\":\"brave_search\",\"content\":\"\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"query\\\\\": \\\\\"Bill Cosby South Park episode\\\\\", \\\\\"top_k\\\\\": \u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Bill Cosby and Taylor Swift Duet - South Park Studios\\\\\", \\\\\"url\\\\\": \\\\\"https://www.southparkstudios.com/video-clips/90r7i1/south-park-bill-cosby-and-taylor-swift-duet\\\\\", \\\\\"content\\\\\": \\\\\"01:05 Bill Cosby is Here to See You South ParkS18 E10 ---------------------------------------------------- Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. 01:03 Bill Cosby and Taylor Swift Duet South ParkS18 E10 ------------------------------------------------------- The holiday special continues with Bill Cosby and Taylor Swift\\'s rendition of \\\\\\\\\\\\\"It\\'s Snowing Out There\\\\\\\\\\\\\". 01:31 #WeBelieveInYou South ParkS18 E10 -------------------------------------- With everyone watching, Kyle takes the opportunity to reach out to his brother. 01:47 Watch Your Microaggressions, Bro South ParkS19 E1 ------------------------------------------------------ Cartman\\'s plan to frame PC Principal backfires. South ParkS19 E1 -------------------------------------- After hearing that the PC people have targeted Kyle, Cartman vows to help.\\\\\", \\\\\"score\\\\\": 0.685971, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Bill Cosby is Here to See You - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/video-clips/wfot8s/south-park-bill-cosby-is-here-to-see-you\\\\\", \\\\\"content\\\\\": \\\\\"01:56 It\\'s Not About Music South ParkS18 E9 ------------------------------------------ At home, Randy sees the consequences of Lorde\\'s performance and calls the Record Producer to try and fix it. 01:24 Lorde\\'s Hologram South ParkS18 E9 -------------------------------------- The Record Producer reveals the truth about the music industry... South ParkS18 E9 --------------------------------------------- Randy catches Sharon with Tupac\\'s hologram. 01:37 I\\'ve Got Your Son, Lorde South ParkS18 E10 ----------------------------------------------- The Record Producer takes Stan and Kyle hostage. 01:05 Bill Cosby is Here to See You South ParkS18 E10 ---------------------------------------------------- Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. 01:21 Lorde Is My Dad South ParkS18 E10 -------------------------------------- After trying to confront Cartman Bra, Stan finally reveals the truth about his dad.\\\\\", \\\\\"score\\\\\": 0.6643884, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Bill Cosby \u001b[0m\u001b[32m(\u001b[0m\u001b[32mandroid\u001b[0m\u001b[32m)\u001b[0m\u001b[32m | South Park Character ... - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/wiki/Bill_Cosby_\u001b[0m\u001b[32m(\u001b[0m\u001b[32mandroid\u001b[0m\u001b[32m)\u001b[0m\u001b[32m\\\\\", \\\\\"content\\\\\": \\\\\"Bill Cosby \u001b[0m\u001b[32m(\u001b[0m\u001b[32mandroid\u001b[0m\u001b[32m)\u001b[0m\u001b[32m | South Park Character / Location / User talk etc | Official South Park Studios Wiki Sent back in time to destroy Eric Cartman\\'s Dawson\\'s Creek Trapper Keeper before it manifests into an omnipotent supercomputer that can destroy all humanity, \\\\\\\\\\\\\"Bill Cosby\\\\\\\\\\\\\" is really VSM471, an android or cyborg of some kind engineered by \\'hoomans\\' in the distant future. He fails in his initial missions to infiltrate South Park Elementary\\'s 4th Grade class, destroy the Trapper Keeper or Cartman himself, but with Stan Marsh and Kyle Broflovski\\'s aid, he is able to succeed in preventing his dismal future, and painfully fades from existence. South Park and all related titles, logos and characters are trademarks of Comedy Partners.\\\\\", \\\\\"score\\\\\": 0.5052006, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"\\\\\\\\\\\\\"South Park\\\\\\\\\\\\\" Clubhouses \u001b[0m\u001b[32m(\u001b[0m\u001b[32mTV Episode 1998\u001b[0m\u001b[32m)\u001b[0m\u001b[32m - IMDb\\\\\", \\\\\"url\\\\\": \\\\\"https://www.imdb.com/title/tt0705915/characters/nm0005295\\\\\", \\\\\"content\\\\\": \\\\\"\\\\\\\\\\\\\"South Park\\\\\\\\\\\\\" Clubhouses \u001b[0m\u001b[32m(\u001b[0m\u001b[32mTV Episode 1998\u001b[0m\u001b[32m)\u001b[0m\u001b[32m - Trey Parker as Stan Marsh, Eric Cartman, Phillip, Randy Marsh, Fat Abbot, Mr. Garrison, Mr. Mackey, 3rd Fat Abbot character, Roy, Teenage Boy #1, Clyde, Bill Cosby, Teenage Boy #2 - IMDb Awards & Events Trey Parker: Stan Marsh, Eric Cartman, Phillip, Randy Marsh, Fat Abbot, Mr. Garrison, Mr. Mackey, 3rd Fat Abbot character, Roy, Teenage Boy #1, Clyde, Bill Cosby, Teenage Boy #2 Mr. Garrison : Stan, are you paying attention? Stan : Yes, Mr. Garrison. Stan Marsh : Dare. Stan Marsh : What? Release Dates | Official Sites | Company Credits | Filming & Production | Technical Specs Photo & Video User Lists Related lists from IMDb users 2024 Watched TV Shows\\\\\", \\\\\"score\\\\\": 0.4604593, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Trapper Keeper \u001b[0m\u001b[32m(\u001b[0m\u001b[32mSouth Park\u001b[0m\u001b[32m)\u001b[0m\u001b[32m - Wikipedia\\\\\", \\\\\"url\\\\\": \\\\\"https://en.wikipedia.org/wiki/Trapper_Keeper_\u001b[0m\u001b[32m(\u001b[0m\u001b[32mSouth_Park\u001b[0m\u001b[32m)\u001b[0m\u001b[32m\\\\\", \\\\\"content\\\\\": \\\\\"\\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" is the twelfth episode of the fourth season of the animated television series South Park, and the 60th episode of the series overall. In the episode, a man from the future wants Cartman\\'s new Trapper Keeper, while Mr. Garrison\\'s kindergarten class holds an election for class president with confusing results. It is one of the many South Park episodes that parodies a current event.\u001b[0m\u001b[32m[\u001b[0m\u001b[32m1\u001b[0m\u001b[32m]\u001b[0m\u001b[32m The main plot of the episode involving the Trapper Keeper was written before the election,\u001b[0m\u001b[32m[\u001b[0m\u001b[32m1\u001b[0m\u001b[32m]\u001b[0m\u001b[32m but the subplot is a parody of the controversy surrounding the election\\'s outcome.\u001b[0m\u001b[32m[\u001b[0m\u001b[32m2\u001b[0m\u001b[32m]\u001b[0m\u001b[32m \\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" did not originally feature the election storyline, only a subplot about Ike attending his first day of kindergarten.\u001b[0m\u001b[32m[\u001b[0m\u001b[32m3\u001b[0m\u001b[32m]\u001b[0m\u001b[32m \\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" Full episode at South Park Studios\\\\\", \\\\\"score\\\\\": 0.3839421, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[1;39m]\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[32m'output'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'content: Bill Cosby \u001b[0m\u001b[32m(\u001b[0m\u001b[32mBSM-471\u001b[0m\u001b[32m)\u001b[0m\u001b[32m first appears in the episode \"Trapper Keeper\" \u001b[0m\u001b[32m(\u001b[0m\u001b[32mSeason 4, Episode 12\u001b[0m\u001b[32m)\u001b[0m\u001b[32m of South Park. tool_calls: \u001b[0m\u001b[32m[\u001b[0m\u001b[32m]\u001b[0m\u001b[32m'\u001b[0m\n", + "\u001b[2;32m│ \u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ \u001b[0m\u001b[1;39m{\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[32m'input'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;39m[\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"system\",\"content\":\"You are a helpful assistant. Use search tool to answer the questions. \"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"call_id\":\"8b7294ec-a83f-4798-ad8f-6bed662f08b6\",\"tool_name\":\"brave_search\",\"arguments\":\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"query\":\"NBA Western Conference Finals 2024 teams\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"tool\",\"call_id\":\"8b7294ec-a83f-4798-ad8f-6bed662f08b6\",\"tool_name\":\"brave_search\",\"content\":\"\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"query\\\\\": \\\\\"NBA Western Conference Finals 2024 teams\\\\\", \\\\\"top_k\\\\\": \u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference Finals - Basketball-Reference.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.basketball-reference.com/playoffs/2024-nba-western-conference-finals-mavericks-vs-timberwolves.html\\\\\", \\\\\"content\\\\\": \\\\\"2024 NBA Western Conference Finals Mavericks vs. Timberwolves League Champion: Boston Celtics. Finals MVP: Jaylen Brown \u001b[0m\u001b[32m(\u001b[0m\u001b[32m20.8 / 5.4 / 5.0\u001b[0m\u001b[32m)\u001b[0m\u001b[32m 2024 Playoff Leaders: PTS: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m635\u001b[0m\u001b[32m)\u001b[0m\u001b[32m TRB: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m208\u001b[0m\u001b[32m)\u001b[0m\u001b[32m AST: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m178\u001b[0m\u001b[32m)\u001b[0m\u001b[32m WS: Derrick White \u001b[0m\u001b[32m(\u001b[0m\u001b[32m2.9\u001b[0m\u001b[32m)\u001b[0m\u001b[32m More playoffs info\\\\\", \\\\\"score\\\\\": 0.9310187, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates, schedule and more - Sportskeeda\\\\\", \\\\\"url\\\\\": \\\\\"https://www.sportskeeda.com/basketball/news-nba-western-conference-finals-2024-dates-schedule-and-more\\\\\", \\\\\"content\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates & Schedule The 2023-24 NBA Western Conference Finals will start on Wednesday, May 22. The Mavericks will face the team that wins in Game 7 between the\\\\\", \\\\\"score\\\\\": 0.8914433, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"2024 Playoffs: West Finals | Timberwolves \u001b[0m\u001b[32m(\u001b[0m\u001b[32m3\u001b[0m\u001b[32m)\u001b[0m\u001b[32m vs. Mavericks \u001b[0m\u001b[32m(\u001b[0m\u001b[32m5\u001b[0m\u001b[32m)\u001b[0m\u001b[32m - NBA.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.nba.com/playoffs/2024/west-final\\\\\", \\\\\"content\\\\\": \\\\\"The Dallas Mavericks and Minnesota Timberwolves have advanced to the 2024 Western Conference Finals during the NBA playoffs.\\\\\", \\\\\"score\\\\\": 0.8884594, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference playoff bracket - Basketnews.com\\\\\", \\\\\"url\\\\\": \\\\\"https://basketnews.com/news-204687-2024-nba-western-conference-playoff-bracket.html\\\\\", \\\\\"content\\\\\": \\\\\"In the 2024 NBA Western Conference playoffs, the Oklahoma City Thunder clinched the No. 1 seed. Every team from the Western Conference played their final game of the regular season, and two playoff pairs have been confirmed. The Los Angeles Lakers beat the New Orleans Pelicans, 110-106, in the Play-In Tournament to secure the 7th seed to set up a first-round matchup with the Denver Nuggets. Meanwhile, the Sacramento Kings will host the Golden State Warriors in the second Western Conference NBA Play-In Tournament game. The winners secure the No. 8 seed in the NBA playoffs for its conference. EuroLeague Play-In: Baskonia-Virtus game schedule announced\\\\\", \\\\\"score\\\\\": 0.8479807, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"NBA Finals 2024 - Celtics-Mavericks news, schedule, scores and ... - ESPN\\\\\", \\\\\"url\\\\\": \\\\\"https://www.espn.com/nba/story/_/id/39943302/nba-playoffs-2024-conference-finals-news-scores-highlights\\\\\", \\\\\"content\\\\\": \\\\\"The Boston Celtics are the 2024 NBA Champions. ... Western Conference. Final 2023-24 NBA regular-season standings. Which team left standing has the most trips to the NBA Finals? Here is a look at\\\\\", \\\\\"score\\\\\": 0.81979275, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"The teams that played in the NBA Western Conference Finals of 2024 were the Dallas Mavericks and the Minnesota Timberwolves.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"user\",\"content\":\"In which episode and season of South Park does Bill Cosby \u001b[0m\u001b[32m(\u001b[0m\u001b[32mBSM-471\u001b[0m\u001b[32m)\u001b[0m\u001b[32m first appear? Give me the number and title.\",\"context\":null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"call_id\":\"fc0441bf-05ad-48d0-8034-4e19cb835904\",\"tool_name\":\"brave_search\",\"arguments\":\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"query\":\"Bill Cosby South Park episode\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"tool\",\"call_id\":\"fc0441bf-05ad-48d0-8034-4e19cb835904\",\"tool_name\":\"brave_search\",\"content\":\"\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"query\\\\\": \\\\\"Bill Cosby South Park episode\\\\\", \\\\\"top_k\\\\\": \u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Bill Cosby and Taylor Swift Duet - South Park Studios\\\\\", \\\\\"url\\\\\": \\\\\"https://www.southparkstudios.com/video-clips/90r7i1/south-park-bill-cosby-and-taylor-swift-duet\\\\\", \\\\\"content\\\\\": \\\\\"01:05 Bill Cosby is Here to See You South ParkS18 E10 ---------------------------------------------------- Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. 01:03 Bill Cosby and Taylor Swift Duet South ParkS18 E10 ------------------------------------------------------- The holiday special continues with Bill Cosby and Taylor Swift\\'s rendition of \\\\\\\\\\\\\"It\\'s Snowing Out There\\\\\\\\\\\\\". 01:31 #WeBelieveInYou South ParkS18 E10 -------------------------------------- With everyone watching, Kyle takes the opportunity to reach out to his brother. 01:47 Watch Your Microaggressions, Bro South ParkS19 E1 ------------------------------------------------------ Cartman\\'s plan to frame PC Principal backfires. South ParkS19 E1 -------------------------------------- After hearing that the PC people have targeted Kyle, Cartman vows to help.\\\\\", \\\\\"score\\\\\": 0.685971, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Bill Cosby is Here to See You - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/video-clips/wfot8s/south-park-bill-cosby-is-here-to-see-you\\\\\", \\\\\"content\\\\\": \\\\\"01:56 It\\'s Not About Music South ParkS18 E9 ------------------------------------------ At home, Randy sees the consequences of Lorde\\'s performance and calls the Record Producer to try and fix it. 01:24 Lorde\\'s Hologram South ParkS18 E9 -------------------------------------- The Record Producer reveals the truth about the music industry... South ParkS18 E9 --------------------------------------------- Randy catches Sharon with Tupac\\'s hologram. 01:37 I\\'ve Got Your Son, Lorde South ParkS18 E10 ----------------------------------------------- The Record Producer takes Stan and Kyle hostage. 01:05 Bill Cosby is Here to See You South ParkS18 E10 ---------------------------------------------------- Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. 01:21 Lorde Is My Dad South ParkS18 E10 -------------------------------------- After trying to confront Cartman Bra, Stan finally reveals the truth about his dad.\\\\\", \\\\\"score\\\\\": 0.6643884, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Bill Cosby \u001b[0m\u001b[32m(\u001b[0m\u001b[32mandroid\u001b[0m\u001b[32m)\u001b[0m\u001b[32m | South Park Character ... - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/wiki/Bill_Cosby_\u001b[0m\u001b[32m(\u001b[0m\u001b[32mandroid\u001b[0m\u001b[32m)\u001b[0m\u001b[32m\\\\\", \\\\\"content\\\\\": \\\\\"Bill Cosby \u001b[0m\u001b[32m(\u001b[0m\u001b[32mandroid\u001b[0m\u001b[32m)\u001b[0m\u001b[32m | South Park Character / Location / User talk etc | Official South Park Studios Wiki Sent back in time to destroy Eric Cartman\\'s Dawson\\'s Creek Trapper Keeper before it manifests into an omnipotent supercomputer that can destroy all humanity, \\\\\\\\\\\\\"Bill Cosby\\\\\\\\\\\\\" is really VSM471, an android or cyborg of some kind engineered by \\'hoomans\\' in the distant future. He fails in his initial missions to infiltrate South Park Elementary\\'s 4th Grade class, destroy the Trapper Keeper or Cartman himself, but with Stan Marsh and Kyle Broflovski\\'s aid, he is able to succeed in preventing his dismal future, and painfully fades from existence. South Park and all related titles, logos and characters are trademarks of Comedy Partners.\\\\\", \\\\\"score\\\\\": 0.5052006, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"\\\\\\\\\\\\\"South Park\\\\\\\\\\\\\" Clubhouses \u001b[0m\u001b[32m(\u001b[0m\u001b[32mTV Episode 1998\u001b[0m\u001b[32m)\u001b[0m\u001b[32m - IMDb\\\\\", \\\\\"url\\\\\": \\\\\"https://www.imdb.com/title/tt0705915/characters/nm0005295\\\\\", \\\\\"content\\\\\": \\\\\"\\\\\\\\\\\\\"South Park\\\\\\\\\\\\\" Clubhouses \u001b[0m\u001b[32m(\u001b[0m\u001b[32mTV Episode 1998\u001b[0m\u001b[32m)\u001b[0m\u001b[32m - Trey Parker as Stan Marsh, Eric Cartman, Phillip, Randy Marsh, Fat Abbot, Mr. Garrison, Mr. Mackey, 3rd Fat Abbot character, Roy, Teenage Boy #1, Clyde, Bill Cosby, Teenage Boy #2 - IMDb Awards & Events Trey Parker: Stan Marsh, Eric Cartman, Phillip, Randy Marsh, Fat Abbot, Mr. Garrison, Mr. Mackey, 3rd Fat Abbot character, Roy, Teenage Boy #1, Clyde, Bill Cosby, Teenage Boy #2 Mr. Garrison : Stan, are you paying attention? Stan : Yes, Mr. Garrison. Stan Marsh : Dare. Stan Marsh : What? Release Dates | Official Sites | Company Credits | Filming & Production | Technical Specs Photo & Video User Lists Related lists from IMDb users 2024 Watched TV Shows\\\\\", \\\\\"score\\\\\": 0.4604593, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Trapper Keeper \u001b[0m\u001b[32m(\u001b[0m\u001b[32mSouth Park\u001b[0m\u001b[32m)\u001b[0m\u001b[32m - Wikipedia\\\\\", \\\\\"url\\\\\": \\\\\"https://en.wikipedia.org/wiki/Trapper_Keeper_\u001b[0m\u001b[32m(\u001b[0m\u001b[32mSouth_Park\u001b[0m\u001b[32m)\u001b[0m\u001b[32m\\\\\", \\\\\"content\\\\\": \\\\\"\\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" is the twelfth episode of the fourth season of the animated television series South Park, and the 60th episode of the series overall. In the episode, a man from the future wants Cartman\\'s new Trapper Keeper, while Mr. Garrison\\'s kindergarten class holds an election for class president with confusing results. It is one of the many South Park episodes that parodies a current event.\u001b[0m\u001b[32m[\u001b[0m\u001b[32m1\u001b[0m\u001b[32m]\u001b[0m\u001b[32m The main plot of the episode involving the Trapper Keeper was written before the election,\u001b[0m\u001b[32m[\u001b[0m\u001b[32m1\u001b[0m\u001b[32m]\u001b[0m\u001b[32m but the subplot is a parody of the controversy surrounding the election\\'s outcome.\u001b[0m\u001b[32m[\u001b[0m\u001b[32m2\u001b[0m\u001b[32m]\u001b[0m\u001b[32m \\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" did not originally feature the election storyline, only a subplot about Ike attending his first day of kindergarten.\u001b[0m\u001b[32m[\u001b[0m\u001b[32m3\u001b[0m\u001b[32m]\u001b[0m\u001b[32m \\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" Full episode at South Park Studios\\\\\", \\\\\"score\\\\\": 0.3839421, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"Bill Cosby \u001b[0m\u001b[32m(\u001b[0m\u001b[32mBSM-471\u001b[0m\u001b[32m)\u001b[0m\u001b[32m first appears in the episode \\\\\"Trapper Keeper\\\\\" \u001b[0m\u001b[32m(\u001b[0m\u001b[32mSeason 4, Episode 12\u001b[0m\u001b[32m)\u001b[0m\u001b[32m of South Park.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"user\",\"content\":\"What is the British-American kickboxer Andrew Tate\\'s kickboxing name?\",\"context\":null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[1m]\u001b[0m,\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'output'\u001b[0m: \u001b[32m\"content: tool_calls: \u001b[0m\u001b[32m[\u001b[0m\u001b[32mToolCall\u001b[0m\u001b[32m(\u001b[0m\u001b[32mcall_id\u001b[0m\u001b[32m='526045a7-5f51-40fb-ba97-5ad29610e511', \u001b[0m\u001b[32mtool_name\u001b[0m\u001b[32m=\u001b[0m\u001b[32m<\u001b[0m\u001b[32mBuiltinTool.brave_search:\u001b[0m\u001b[32m 'brave_search'\u001b[0m\u001b[32m>\u001b[0m\u001b[32m, \u001b[0m\u001b[32marguments\u001b[0m\u001b[32m=\u001b[0m\u001b[32m{\u001b[0m\u001b[32m'query': 'Andrew Tate kickboxing name'\u001b[0m\u001b[32m}\u001b[0m\u001b[32m)\u001b[0m\u001b[32m]\u001b[0m\u001b[32m\"\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[1;39m]\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[32m'output'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m\"content: tool_calls: \u001b[0m\u001b[32m[\u001b[0m\u001b[32mToolCall\u001b[0m\u001b[32m(\u001b[0m\u001b[32mcall_id\u001b[0m\u001b[32m='79276f65-3600-489d-ab41-d5a71dcaf075', \u001b[0m\u001b[32mtool_name\u001b[0m\u001b[32m=\u001b[0m\u001b[32m, \u001b[0m\u001b[32marguments\u001b[0m\u001b[32m=\u001b[0m\u001b[32m{\u001b[0m\u001b[32m'query': 'Andrew Tate kickboxing name'\u001b[0m\u001b[32m}\u001b[0m\u001b[32m)\u001b[0m\u001b[32m]\u001b[0m\u001b[32m\"\u001b[0m\n", "\u001b[2;32m│ \u001b[0m\u001b[1m}\u001b[0m,\n", "\u001b[2;32m│ \u001b[0m\u001b[1m{\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'input'\u001b[0m: \u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"call_id\":\"526045a7-5f51-40fb-ba97-5ad29610e511\",\"tool_name\":\"brave_search\",\"arguments\":\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"query\":\"Andrew Tate kickboxing name\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'output'\u001b[0m: \u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"ipython\",\"call_id\":\"526045a7-5f51-40fb-ba97-5ad29610e511\",\"tool_name\":\"brave_search\",\"content\":\"\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"query\\\\\": \\\\\"Andrew Tate kickboxing name\\\\\", \\\\\"top_k\\\\\": \u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Andrew Tate kickboxing record: How many championships ... - FirstSportz\\\\\", \\\\\"url\\\\\": \\\\\"https://firstsportz.com/mma-how-many-championships-does-andrew-tate-have/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate\\'s Kickboxing career. During his kickboxing career, he used the nickname \\\\\\\\\\\\\"King Cobra,\\\\\\\\\\\\\" which he currently uses as his Twitter name. Tate had an unorthodox style of movement inside the ring. He kept his hands down most of the time and relied on quick jabs and an overhand right to land significant strikes.\\\\\", \\\\\"score\\\\\": 0.9996244, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Andrew Tate: Kickboxing Record, Facts, Height, Weight, Age, Biography\\\\\", \\\\\"url\\\\\": \\\\\"https://www.lowkickmma.com/andrew-tate-kickboxing-record-facts-height-weight-age-biography/\\\\\", \\\\\"content\\\\\": \\\\\"Birth Name: Emory Andrew Tate III: Date of Birth: 1 December 1986: Place of Birth: Washington, D.C., U.S. ... In his professional kickboxing career, Andrew Tate won 32 of his fights by knockout.\\\\\", \\\\\"score\\\\\": 0.99909246, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Who is Andrew Tate? MMA, kickboxing record and controversies of fighter ...\\\\\", \\\\\"url\\\\\": \\\\\"https://www.sportingnews.com/us/kickboxing/news/andrew-tate-mma-kickboxing-record-controversies/u50waalc9cfz7krjg9wnyb7p\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate kickboxing record After launching his career as a 20-year-old in 2007, Tate built a formidable kickboxing record that included 76 wins across 85 fights in more than 13 years in the ring.\\\\\", \\\\\"score\\\\\": 0.9976586, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"About Andrew Tate: A Journey from Champion to Controversy\\\\\", \\\\\"url\\\\\": \\\\\"https://reachmorpheus.com/andrew-tate/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate\\'s kickboxing career, beginning in 2005, is a tale of determination and skill. He quickly made a name for himself in the sport, rising through the ranks with his unique fighting style and strategic approach, honed by his chess-playing background.\\\\\", \\\\\"score\\\\\": 0.99701905, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Andrew Tate Bio, Wiki, Net Worth, Age, Family, MMA Career - Next Biography\\\\\", \\\\\"url\\\\\": \\\\\"https://www.nextbiography.com/andrew-tate/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate Age. Andrew Tate is 36 years old as of 2023, born on December 1, 1986, in Washington, DC. By his mid-thirties, Andrew Tate has become an esteemed figure in the world of kickboxing, showcasing remarkable expertise and experience in the sport. Early Life of Andrew Tate. Andrew Tate was born on 01 December 1986 to an African-American\\\\\", \\\\\"score\\\\\": 0.99368566, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[32m'input'\u001b[0m: \u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"call_id\":\"79276f65-3600-489d-ab41-d5a71dcaf075\",\"tool_name\":\"brave_search\",\"arguments\":\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"query\":\"Andrew Tate kickboxing name\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[32m'output'\u001b[0m: \u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"tool\",\"call_id\":\"79276f65-3600-489d-ab41-d5a71dcaf075\",\"tool_name\":\"brave_search\",\"content\":\"\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"query\\\\\": \\\\\"Andrew Tate kickboxing name\\\\\", \\\\\"top_k\\\\\": \u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Andrew Tate Age, Height, Weight, Family, Parents, Biography, Net Worth\\\\\", \\\\\"url\\\\\": \\\\\"https://biographywallah.com/andrew-tate-biography/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate Age, Height, Weight, Family, Parents, Biography, Net Worth \\\\\\\\u00bb Biography Wallah Andrew Tate Age, Height, Weight, Family, Parents, Biography, Net Worth Andrew Tate Biography NameAndrew TateReal nameEmory Andrew Tate IIIProfession \\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0Kickboxer, Commentator and BusinessmanDate of birth14 December 1986BirthplaceWashington D.C., United StatesAndrew Tate Age37 years old \u001b[0m\u001b[32m(\u001b[0m\u001b[32mas of 2024\u001b[0m\u001b[32m)\u001b[0m\u001b[32mNationalityBritish-AmericanZodiac SignSagittariusGenderMaleSchoolLocal School in Washington D.C., United StatesGirlfriend/SpouseNaghel GeorgianaSexual OrientationStraightNet worth$1000 Million Who is Andrew Tate? Andrew Tate is a British-American former professional kickboxing world champion businessman and media personality, who fought in the cruiserweight and super cruiserweight divisions. Andrew Tate Age Andrew Tate was born on 1 December 1986 and is 37 years old. Andrew Tate\\\\\\\\u2019s Net Worth What is the net worth of Andrew Tate? Where is Andrew Tate from? How old is Andrew Tate?\\\\\", \\\\\"score\\\\\": 0.80698997, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"The Life Of Andrew Tate \u001b[0m\u001b[32m(\u001b[0m\u001b[32mBy Andrew Tate Himself ... - Sidekick Boxing\\\\\", \\\\\"url\\\\\": \\\\\"https://sidekickboxing.co.uk/the-life-of-andrew-king-cobra-tate/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate is a British-American former professional kickboxing world champion who fought in the cruiserweight and super cruiserweight divisions. Andrew Tate\\\\\\\\u2019s Kickboxing Career Andrew Tate in the Big Brother house Andrew Tate\\\\\\\\u2019s Kickboxing World Titles and his Sidekick boxing gloves Andrew Tate After Kickboxing Andrew Tate and his brother Tristan moved to Romania to set up their empire of businesses including trading in Bitcoin, Hustlers University, CobraTate.com, The Real World, and The War Room. From being a 4x kickboxing world champion to becoming the world\\\\\\\\u2019s most Googled man in the world with a private jet and over 33 cars, Andrew Tate\\\\\\\\u2019s life has been full of adventure.\\\\\", \\\\\"score\\\\\": 0.78194773, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Andrew Tate \u001b[0m\u001b[32m(\u001b[0m\u001b[32m\\\\\\\\\\\\\"King Cobra\\\\\\\\\\\\\"\u001b[0m\u001b[32m)\u001b[0m\u001b[32m | MMA Fighter Page - Tapology\\\\\", \\\\\"url\\\\\": \\\\\"https://www.tapology.com/fightcenter/fighters/72139-andrew-tate\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate \u001b[0m\u001b[32m(\u001b[0m\u001b[32m\\\\\\\\\\\\\"King Cobra\\\\\\\\\\\\\"\u001b[0m\u001b[32m)\u001b[0m\u001b[32m | MMA Fighter Page | Tapology Andrew \\\\\\\\\\\\\"King Cobra\\\\\\\\\\\\\" Tate Andrew Tate Name: Andrew Tate Height: 6\\'1\\\\\\\\\\\\\" \u001b[0m\u001b[32m(\u001b[0m\u001b[32m185cm\u001b[0m\u001b[32m)\u001b[0m\u001b[32m | Reach: Andrew Tate is ineligible for Tapology\\'s regional MMA rankings due to inactivity. Fighters must have at least one completed MMA bout in the past two years to be ranked. Andrew Tate MMA Fight Record Former top-ranked UFC fighter has called out Andrew Tate for having a paper title when it comes to combat... Andrew Tate \\\\\\\\u2022 All the biggest upcoming MMA & Boxing fights | UFC Fight Night | 02.01.2025, 12:00 PM ET | MMA Junkie: UFC Fight Night 249 video: Nine stoppages to open the year?! MMA Mania: Prochazka Vs. Hill: Odds, Full Fight Preview & Prediction\\\\\", \\\\\"score\\\\\": 0.6999322, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"About Andrew Tate: A Journey from Champion to Controversy\\\\\", \\\\\"url\\\\\": \\\\\"https://reachmorpheus.com/andrew-tate/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate\\'s kickboxing career, beginning in 2005, is a tale of determination and skill. He quickly made a name for himself in the sport, rising through the ranks with his unique fighting style and strategic approach, honed by his chess-playing background.\\\\\", \\\\\"score\\\\\": 0.6490677, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Andrew Tate\\'s Kickboxing Career & Biography - MMA Full Contact\\\\\", \\\\\"url\\\\\": \\\\\"https://www.mmafullcontact.com/andrew-tate-kickboxing/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate\\'s Kickboxing Career & Biography - MMA Full Contact Andrew Tate\\\\\\\\u2019s Kickboxing Career & Biography 2 Notable Opponents and Fights in Andrew Tate\\\\\\\\u2019s Kickboxing Career 4 Will Andrew Tate fight KSI? Notable Opponents and Fights in Andrew Tate\\\\\\\\u2019s Kickboxing Career Will Andrew Tate fight KSI? Similarly, Andrew Tate, known for his successful kickboxing career, has also shown interest in a potential fight with KSI. In conclusion, while there\\\\\\\\u2019s been plenty of interest and discussion about a potential boxing match between KSI and Andrew Tate, no official confirmation has been made as of now. With KSI\\\\\\\\u2019s upcoming match and Tate\\\\\\\\u2019s current personal circumstances, fans and followers of both personalities will have to wait for more updates on this potential fight.\\\\\", \\\\\"score\\\\\": 0.53050464, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\n", "\u001b[2;32m│ \u001b[0m\u001b[1m}\u001b[0m,\n", "\u001b[2;32m│ \u001b[0m\u001b[1m{\u001b[0m\n", "\u001b[2;32m│ │ \u001b[0m\u001b[32m'input'\u001b[0m: \u001b[1m[\u001b[0m\n", "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"system\",\"content\":\"You are a helpful assistant. Use search tool to answer the questions. \"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"Let me check the latest sports news.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"call_id\":\"8b7294ec-a83f-4798-ad8f-6bed662f08b6\",\"tool_name\":\"brave_search\",\"arguments\":\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"query\":\"NBA Western Conference Finals 2024 teams\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"tool\",\"call_id\":\"8b7294ec-a83f-4798-ad8f-6bed662f08b6\",\"tool_name\":\"brave_search\",\"content\":\"\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"query\\\\\": \\\\\"NBA Western Conference Finals 2024 teams\\\\\", \\\\\"top_k\\\\\": \u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference Finals - Basketball-Reference.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.basketball-reference.com/playoffs/2024-nba-western-conference-finals-mavericks-vs-timberwolves.html\\\\\", \\\\\"content\\\\\": \\\\\"2024 NBA Western Conference Finals Mavericks vs. Timberwolves League Champion: Boston Celtics. Finals MVP: Jaylen Brown \u001b[0m\u001b[32m(\u001b[0m\u001b[32m20.8 / 5.4 / 5.0\u001b[0m\u001b[32m)\u001b[0m\u001b[32m 2024 Playoff Leaders: PTS: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m635\u001b[0m\u001b[32m)\u001b[0m\u001b[32m TRB: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m208\u001b[0m\u001b[32m)\u001b[0m\u001b[32m AST: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m178\u001b[0m\u001b[32m)\u001b[0m\u001b[32m WS: Derrick White \u001b[0m\u001b[32m(\u001b[0m\u001b[32m2.9\u001b[0m\u001b[32m)\u001b[0m\u001b[32m More playoffs info\\\\\", \\\\\"score\\\\\": 0.9310187, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates, schedule and more - Sportskeeda\\\\\", \\\\\"url\\\\\": \\\\\"https://www.sportskeeda.com/basketball/news-nba-western-conference-finals-2024-dates-schedule-and-more\\\\\", \\\\\"content\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates & Schedule The 2023-24 NBA Western Conference Finals will start on Wednesday, May 22. The Mavericks will face the team that wins in Game 7 between the\\\\\", \\\\\"score\\\\\": 0.8914433, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"2024 Playoffs: West Finals | Timberwolves \u001b[0m\u001b[32m(\u001b[0m\u001b[32m3\u001b[0m\u001b[32m)\u001b[0m\u001b[32m vs. Mavericks \u001b[0m\u001b[32m(\u001b[0m\u001b[32m5\u001b[0m\u001b[32m)\u001b[0m\u001b[32m - NBA.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.nba.com/playoffs/2024/west-final\\\\\", \\\\\"content\\\\\": \\\\\"The Dallas Mavericks and Minnesota Timberwolves have advanced to the 2024 Western Conference Finals during the NBA playoffs.\\\\\", \\\\\"score\\\\\": 0.8884594, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference playoff bracket - Basketnews.com\\\\\", \\\\\"url\\\\\": \\\\\"https://basketnews.com/news-204687-2024-nba-western-conference-playoff-bracket.html\\\\\", \\\\\"content\\\\\": \\\\\"In the 2024 NBA Western Conference playoffs, the Oklahoma City Thunder clinched the No. 1 seed. Every team from the Western Conference played their final game of the regular season, and two playoff pairs have been confirmed. The Los Angeles Lakers beat the New Orleans Pelicans, 110-106, in the Play-In Tournament to secure the 7th seed to set up a first-round matchup with the Denver Nuggets. Meanwhile, the Sacramento Kings will host the Golden State Warriors in the second Western Conference NBA Play-In Tournament game. The winners secure the No. 8 seed in the NBA playoffs for its conference. EuroLeague Play-In: Baskonia-Virtus game schedule announced\\\\\", \\\\\"score\\\\\": 0.8479807, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"NBA Finals 2024 - Celtics-Mavericks news, schedule, scores and ... - ESPN\\\\\", \\\\\"url\\\\\": \\\\\"https://www.espn.com/nba/story/_/id/39943302/nba-playoffs-2024-conference-finals-news-scores-highlights\\\\\", \\\\\"content\\\\\": \\\\\"The Boston Celtics are the 2024 NBA Champions. ... Western Conference. Final 2023-24 NBA regular-season standings. Which team left standing has the most trips to the NBA Finals? Here is a look at\\\\\", \\\\\"score\\\\\": 0.81979275, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"The teams that played in the NBA Western Conference Finals of 2024 were the Dallas Mavericks and the Minnesota Timberwolves.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"user\",\"content\":\"In which episode and season of South Park does Bill Cosby \u001b[0m\u001b[32m(\u001b[0m\u001b[32mBSM-471\u001b[0m\u001b[32m)\u001b[0m\u001b[32m first appear? Give me the number and title.\",\"context\":null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"call_id\":\"19bd3554-e670-4856-89d0-c63f5b016245\",\"tool_name\":\"bravy_search\",\"arguments\":\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"query\":\"Bill Cosby South Park episode\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"call_id\":\"fc0441bf-05ad-48d0-8034-4e19cb835904\",\"tool_name\":\"brave_search\",\"arguments\":\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"query\":\"Bill Cosby South Park episode\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"tool\",\"call_id\":\"fc0441bf-05ad-48d0-8034-4e19cb835904\",\"tool_name\":\"brave_search\",\"content\":\"\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"query\\\\\": \\\\\"Bill Cosby South Park episode\\\\\", \\\\\"top_k\\\\\": \u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Bill Cosby and Taylor Swift Duet - South Park Studios\\\\\", \\\\\"url\\\\\": \\\\\"https://www.southparkstudios.com/video-clips/90r7i1/south-park-bill-cosby-and-taylor-swift-duet\\\\\", \\\\\"content\\\\\": \\\\\"01:05 Bill Cosby is Here to See You South ParkS18 E10 ---------------------------------------------------- Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. 01:03 Bill Cosby and Taylor Swift Duet South ParkS18 E10 ------------------------------------------------------- The holiday special continues with Bill Cosby and Taylor Swift\\'s rendition of \\\\\\\\\\\\\"It\\'s Snowing Out There\\\\\\\\\\\\\". 01:31 #WeBelieveInYou South ParkS18 E10 -------------------------------------- With everyone watching, Kyle takes the opportunity to reach out to his brother. 01:47 Watch Your Microaggressions, Bro South ParkS19 E1 ------------------------------------------------------ Cartman\\'s plan to frame PC Principal backfires. South ParkS19 E1 -------------------------------------- After hearing that the PC people have targeted Kyle, Cartman vows to help.\\\\\", \\\\\"score\\\\\": 0.685971, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Bill Cosby is Here to See You - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/video-clips/wfot8s/south-park-bill-cosby-is-here-to-see-you\\\\\", \\\\\"content\\\\\": \\\\\"01:56 It\\'s Not About Music South ParkS18 E9 ------------------------------------------ At home, Randy sees the consequences of Lorde\\'s performance and calls the Record Producer to try and fix it. 01:24 Lorde\\'s Hologram South ParkS18 E9 -------------------------------------- The Record Producer reveals the truth about the music industry... South ParkS18 E9 --------------------------------------------- Randy catches Sharon with Tupac\\'s hologram. 01:37 I\\'ve Got Your Son, Lorde South ParkS18 E10 ----------------------------------------------- The Record Producer takes Stan and Kyle hostage. 01:05 Bill Cosby is Here to See You South ParkS18 E10 ---------------------------------------------------- Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. 01:21 Lorde Is My Dad South ParkS18 E10 -------------------------------------- After trying to confront Cartman Bra, Stan finally reveals the truth about his dad.\\\\\", \\\\\"score\\\\\": 0.6643884, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Bill Cosby \u001b[0m\u001b[32m(\u001b[0m\u001b[32mandroid\u001b[0m\u001b[32m)\u001b[0m\u001b[32m | South Park Character ... - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/wiki/Bill_Cosby_\u001b[0m\u001b[32m(\u001b[0m\u001b[32mandroid\u001b[0m\u001b[32m)\u001b[0m\u001b[32m\\\\\", \\\\\"content\\\\\": \\\\\"Bill Cosby \u001b[0m\u001b[32m(\u001b[0m\u001b[32mandroid\u001b[0m\u001b[32m)\u001b[0m\u001b[32m | South Park Character / Location / User talk etc | Official South Park Studios Wiki Sent back in time to destroy Eric Cartman\\'s Dawson\\'s Creek Trapper Keeper before it manifests into an omnipotent supercomputer that can destroy all humanity, \\\\\\\\\\\\\"Bill Cosby\\\\\\\\\\\\\" is really VSM471, an android or cyborg of some kind engineered by \\'hoomans\\' in the distant future. He fails in his initial missions to infiltrate South Park Elementary\\'s 4th Grade class, destroy the Trapper Keeper or Cartman himself, but with Stan Marsh and Kyle Broflovski\\'s aid, he is able to succeed in preventing his dismal future, and painfully fades from existence. South Park and all related titles, logos and characters are trademarks of Comedy Partners.\\\\\", \\\\\"score\\\\\": 0.5052006, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"\\\\\\\\\\\\\"South Park\\\\\\\\\\\\\" Clubhouses \u001b[0m\u001b[32m(\u001b[0m\u001b[32mTV Episode 1998\u001b[0m\u001b[32m)\u001b[0m\u001b[32m - IMDb\\\\\", \\\\\"url\\\\\": \\\\\"https://www.imdb.com/title/tt0705915/characters/nm0005295\\\\\", \\\\\"content\\\\\": \\\\\"\\\\\\\\\\\\\"South Park\\\\\\\\\\\\\" Clubhouses \u001b[0m\u001b[32m(\u001b[0m\u001b[32mTV Episode 1998\u001b[0m\u001b[32m)\u001b[0m\u001b[32m - Trey Parker as Stan Marsh, Eric Cartman, Phillip, Randy Marsh, Fat Abbot, Mr. Garrison, Mr. Mackey, 3rd Fat Abbot character, Roy, Teenage Boy #1, Clyde, Bill Cosby, Teenage Boy #2 - IMDb Awards & Events Trey Parker: Stan Marsh, Eric Cartman, Phillip, Randy Marsh, Fat Abbot, Mr. Garrison, Mr. Mackey, 3rd Fat Abbot character, Roy, Teenage Boy #1, Clyde, Bill Cosby, Teenage Boy #2 Mr. Garrison : Stan, are you paying attention? Stan : Yes, Mr. Garrison. Stan Marsh : Dare. Stan Marsh : What? Release Dates | Official Sites | Company Credits | Filming & Production | Technical Specs Photo & Video User Lists Related lists from IMDb users 2024 Watched TV Shows\\\\\", \\\\\"score\\\\\": 0.4604593, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Trapper Keeper \u001b[0m\u001b[32m(\u001b[0m\u001b[32mSouth Park\u001b[0m\u001b[32m)\u001b[0m\u001b[32m - Wikipedia\\\\\", \\\\\"url\\\\\": \\\\\"https://en.wikipedia.org/wiki/Trapper_Keeper_\u001b[0m\u001b[32m(\u001b[0m\u001b[32mSouth_Park\u001b[0m\u001b[32m)\u001b[0m\u001b[32m\\\\\", \\\\\"content\\\\\": \\\\\"\\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" is the twelfth episode of the fourth season of the animated television series South Park, and the 60th episode of the series overall. In the episode, a man from the future wants Cartman\\'s new Trapper Keeper, while Mr. Garrison\\'s kindergarten class holds an election for class president with confusing results. It is one of the many South Park episodes that parodies a current event.\u001b[0m\u001b[32m[\u001b[0m\u001b[32m1\u001b[0m\u001b[32m]\u001b[0m\u001b[32m The main plot of the episode involving the Trapper Keeper was written before the election,\u001b[0m\u001b[32m[\u001b[0m\u001b[32m1\u001b[0m\u001b[32m]\u001b[0m\u001b[32m but the subplot is a parody of the controversy surrounding the election\\'s outcome.\u001b[0m\u001b[32m[\u001b[0m\u001b[32m2\u001b[0m\u001b[32m]\u001b[0m\u001b[32m \\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" did not originally feature the election storyline, only a subplot about Ike attending his first day of kindergarten.\u001b[0m\u001b[32m[\u001b[0m\u001b[32m3\u001b[0m\u001b[32m]\u001b[0m\u001b[32m \\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" Full episode at South Park Studios\\\\\", \\\\\"score\\\\\": 0.3839421, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"Bill Cosby \u001b[0m\u001b[32m(\u001b[0m\u001b[32mBSM-471\u001b[0m\u001b[32m)\u001b[0m\u001b[32m first appears in the episode \\\\\"Trapper Keeper\\\\\" \u001b[0m\u001b[32m(\u001b[0m\u001b[32mSeason 4, Episode 12\u001b[0m\u001b[32m)\u001b[0m\u001b[32m of South Park.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"user\",\"content\":\"What is the British-American kickboxer Andrew Tate\\'s kickboxing name?\",\"context\":null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"call_id\":\"526045a7-5f51-40fb-ba97-5ad29610e511\",\"tool_name\":\"brave_search\",\"arguments\":\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"query\":\"Andrew Tate kickboxing name\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"ipython\",\"call_id\":\"526045a7-5f51-40fb-ba97-5ad29610e511\",\"tool_name\":\"brave_search\",\"content\":\"\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"query\\\\\": \\\\\"Andrew Tate kickboxing name\\\\\", \\\\\"top_k\\\\\": \u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Andrew Tate kickboxing record: How many championships ... - FirstSportz\\\\\", \\\\\"url\\\\\": \\\\\"https://firstsportz.com/mma-how-many-championships-does-andrew-tate-have/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate\\'s Kickboxing career. During his kickboxing career, he used the nickname \\\\\\\\\\\\\"King Cobra,\\\\\\\\\\\\\" which he currently uses as his Twitter name. Tate had an unorthodox style of movement inside the ring. He kept his hands down most of the time and relied on quick jabs and an overhand right to land significant strikes.\\\\\", \\\\\"score\\\\\": 0.9996244, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Andrew Tate: Kickboxing Record, Facts, Height, Weight, Age, Biography\\\\\", \\\\\"url\\\\\": \\\\\"https://www.lowkickmma.com/andrew-tate-kickboxing-record-facts-height-weight-age-biography/\\\\\", \\\\\"content\\\\\": \\\\\"Birth Name: Emory Andrew Tate III: Date of Birth: 1 December 1986: Place of Birth: Washington, D.C., U.S. ... In his professional kickboxing career, Andrew Tate won 32 of his fights by knockout.\\\\\", \\\\\"score\\\\\": 0.99909246, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Who is Andrew Tate? MMA, kickboxing record and controversies of fighter ...\\\\\", \\\\\"url\\\\\": \\\\\"https://www.sportingnews.com/us/kickboxing/news/andrew-tate-mma-kickboxing-record-controversies/u50waalc9cfz7krjg9wnyb7p\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate kickboxing record After launching his career as a 20-year-old in 2007, Tate built a formidable kickboxing record that included 76 wins across 85 fights in more than 13 years in the ring.\\\\\", \\\\\"score\\\\\": 0.9976586, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"About Andrew Tate: A Journey from Champion to Controversy\\\\\", \\\\\"url\\\\\": \\\\\"https://reachmorpheus.com/andrew-tate/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate\\'s kickboxing career, beginning in 2005, is a tale of determination and skill. He quickly made a name for himself in the sport, rising through the ranks with his unique fighting style and strategic approach, honed by his chess-playing background.\\\\\", \\\\\"score\\\\\": 0.99701905, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Andrew Tate Bio, Wiki, Net Worth, Age, Family, MMA Career - Next Biography\\\\\", \\\\\"url\\\\\": \\\\\"https://www.nextbiography.com/andrew-tate/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate Age. Andrew Tate is 36 years old as of 2023, born on December 1, 1986, in Washington, DC. By his mid-thirties, Andrew Tate has become an esteemed figure in the world of kickboxing, showcasing remarkable expertise and experience in the sport. Early Life of Andrew Tate. Andrew Tate was born on 01 December 1986 to an African-American\\\\\", \\\\\"score\\\\\": 0.99368566, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"call_id\":\"79276f65-3600-489d-ab41-d5a71dcaf075\",\"tool_name\":\"brave_search\",\"arguments\":\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"query\":\"Andrew Tate kickboxing name\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"tool\",\"call_id\":\"79276f65-3600-489d-ab41-d5a71dcaf075\",\"tool_name\":\"brave_search\",\"content\":\"\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"query\\\\\": \\\\\"Andrew Tate kickboxing name\\\\\", \\\\\"top_k\\\\\": \u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Andrew Tate Age, Height, Weight, Family, Parents, Biography, Net Worth\\\\\", \\\\\"url\\\\\": \\\\\"https://biographywallah.com/andrew-tate-biography/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate Age, Height, Weight, Family, Parents, Biography, Net Worth \\\\\\\\u00bb Biography Wallah Andrew Tate Age, Height, Weight, Family, Parents, Biography, Net Worth Andrew Tate Biography NameAndrew TateReal nameEmory Andrew Tate IIIProfession \\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0Kickboxer, Commentator and BusinessmanDate of birth14 December 1986BirthplaceWashington D.C., United StatesAndrew Tate Age37 years old \u001b[0m\u001b[32m(\u001b[0m\u001b[32mas of 2024\u001b[0m\u001b[32m)\u001b[0m\u001b[32mNationalityBritish-AmericanZodiac SignSagittariusGenderMaleSchoolLocal School in Washington D.C., United StatesGirlfriend/SpouseNaghel GeorgianaSexual OrientationStraightNet worth$1000 Million Who is Andrew Tate? Andrew Tate is a British-American former professional kickboxing world champion businessman and media personality, who fought in the cruiserweight and super cruiserweight divisions. Andrew Tate Age Andrew Tate was born on 1 December 1986 and is 37 years old. Andrew Tate\\\\\\\\u2019s Net Worth What is the net worth of Andrew Tate? Where is Andrew Tate from? How old is Andrew Tate?\\\\\", \\\\\"score\\\\\": 0.80698997, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"The Life Of Andrew Tate \u001b[0m\u001b[32m(\u001b[0m\u001b[32mBy Andrew Tate Himself ... - Sidekick Boxing\\\\\", \\\\\"url\\\\\": \\\\\"https://sidekickboxing.co.uk/the-life-of-andrew-king-cobra-tate/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate is a British-American former professional kickboxing world champion who fought in the cruiserweight and super cruiserweight divisions. Andrew Tate\\\\\\\\u2019s Kickboxing Career Andrew Tate in the Big Brother house Andrew Tate\\\\\\\\u2019s Kickboxing World Titles and his Sidekick boxing gloves Andrew Tate After Kickboxing Andrew Tate and his brother Tristan moved to Romania to set up their empire of businesses including trading in Bitcoin, Hustlers University, CobraTate.com, The Real World, and The War Room. From being a 4x kickboxing world champion to becoming the world\\\\\\\\u2019s most Googled man in the world with a private jet and over 33 cars, Andrew Tate\\\\\\\\u2019s life has been full of adventure.\\\\\", \\\\\"score\\\\\": 0.78194773, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Andrew Tate \u001b[0m\u001b[32m(\u001b[0m\u001b[32m\\\\\\\\\\\\\"King Cobra\\\\\\\\\\\\\"\u001b[0m\u001b[32m)\u001b[0m\u001b[32m | MMA Fighter Page - Tapology\\\\\", \\\\\"url\\\\\": \\\\\"https://www.tapology.com/fightcenter/fighters/72139-andrew-tate\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate \u001b[0m\u001b[32m(\u001b[0m\u001b[32m\\\\\\\\\\\\\"King Cobra\\\\\\\\\\\\\"\u001b[0m\u001b[32m)\u001b[0m\u001b[32m | MMA Fighter Page | Tapology Andrew \\\\\\\\\\\\\"King Cobra\\\\\\\\\\\\\" Tate Andrew Tate Name: Andrew Tate Height: 6\\'1\\\\\\\\\\\\\" \u001b[0m\u001b[32m(\u001b[0m\u001b[32m185cm\u001b[0m\u001b[32m)\u001b[0m\u001b[32m | Reach: Andrew Tate is ineligible for Tapology\\'s regional MMA rankings due to inactivity. Fighters must have at least one completed MMA bout in the past two years to be ranked. Andrew Tate MMA Fight Record Former top-ranked UFC fighter has called out Andrew Tate for having a paper title when it comes to combat... Andrew Tate \\\\\\\\u2022 All the biggest upcoming MMA & Boxing fights | UFC Fight Night | 02.01.2025, 12:00 PM ET | MMA Junkie: UFC Fight Night 249 video: Nine stoppages to open the year?! MMA Mania: Prochazka Vs. Hill: Odds, Full Fight Preview & Prediction\\\\\", \\\\\"score\\\\\": 0.6999322, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"About Andrew Tate: A Journey from Champion to Controversy\\\\\", \\\\\"url\\\\\": \\\\\"https://reachmorpheus.com/andrew-tate/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate\\'s kickboxing career, beginning in 2005, is a tale of determination and skill. He quickly made a name for himself in the sport, rising through the ranks with his unique fighting style and strategic approach, honed by his chess-playing background.\\\\\", \\\\\"score\\\\\": 0.6490677, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Andrew Tate\\'s Kickboxing Career & Biography - MMA Full Contact\\\\\", \\\\\"url\\\\\": \\\\\"https://www.mmafullcontact.com/andrew-tate-kickboxing/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate\\'s Kickboxing Career & Biography - MMA Full Contact Andrew Tate\\\\\\\\u2019s Kickboxing Career & Biography 2 Notable Opponents and Fights in Andrew Tate\\\\\\\\u2019s Kickboxing Career 4 Will Andrew Tate fight KSI? Notable Opponents and Fights in Andrew Tate\\\\\\\\u2019s Kickboxing Career Will Andrew Tate fight KSI? Similarly, Andrew Tate, known for his successful kickboxing career, has also shown interest in a potential fight with KSI. In conclusion, while there\\\\\\\\u2019s been plenty of interest and discussion about a potential boxing match between KSI and Andrew Tate, no official confirmation has been made as of now. With KSI\\\\\\\\u2019s upcoming match and Tate\\\\\\\\u2019s current personal circumstances, fans and followers of both personalities will have to wait for more updates on this potential fight.\\\\\", \\\\\"score\\\\\": 0.53050464, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\n", "\u001b[2;32m│ │ \u001b[0m\u001b[1m]\u001b[0m,\n", "\u001b[2;32m│ │ \u001b[0m\u001b[32m'output'\u001b[0m: \u001b[32m'content: Andrew Tate\\'s kickboxing name is \"King Cobra.\" tool_calls: \u001b[0m\u001b[32m[\u001b[0m\u001b[32m]\u001b[0m\u001b[32m'\u001b[0m\n", "\u001b[2;32m│ \u001b[0m\u001b[1m}\u001b[0m\n", @@ -2263,22 +3523,24 @@ } ], "source": [ + "# NBVAL_SKIP \n", "print(f\"Getting traces for session_id={session_id}\")\n", "import json\n", + "\n", "from rich.pretty import pprint\n", "\n", "agent_logs = []\n", "\n", "for span in client.telemetry.query_spans(\n", " attribute_filters=[\n", - " {\"key\": \"session_id\", \"op\": \"eq\", \"value\": session_id},\n", + " {\"key\": \"session_id\", \"op\": \"eq\", \"value\": session_id},\n", " ],\n", - " attributes_to_return=[\"input\", \"output\"]\n", - " ):\n", - " if span.attributes[\"output\"] != \"no shields\":\n", - " agent_logs.append(span.attributes)\n", + " attributes_to_return=[\"input\", \"output\"],\n", + "):\n", + " if span.attributes[\"output\"] != \"no shields\":\n", + " agent_logs.append(span.attributes)\n", "\n", - "pprint(agent_logs)" + "pprint(agent_logs)\n" ] }, { @@ -2296,17 +3558,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "id": "sy4Xaff_Avuu", "metadata": { "colab": { "base_uri": "https://localhost:8080/", - "height": 411 + "height": 432 }, "id": "sy4Xaff_Avuu", - "outputId": "cb68bae7-b21d-415d-8e71-612bd383c793" + "outputId": "1b14b5ed-4c77-47c4-edfb-1c13a88e5ef4" }, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'input': ['{\"role\":\"system\",\"content\":\"You are a helpful assistant. Use search tool to answer the questions. \"}', '{\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null}'], 'output': 'content: Let me check the latest sports news. tool_calls: []'}\n", + "{'input': ['{\"role\":\"system\",\"content\":\"You are a helpful assistant. Use search tool to answer the questions. \"}', '{\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null}', '{\"role\":\"assistant\",\"content\":\"Let me check the latest sports news.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[]}', '{\"role\":\"user\",\"content\":\"In which episode and season of South Park does Bill Cosby (BSM-471) first appear? Give me the number and title.\",\"context\":null}'], 'output': \"content: tool_calls: [ToolCall(call_id='26345b28-7f75-401e-88e3-77933cb70a2e', tool_name=, arguments={'query': 'Bill Cosby South Park episode'})]\"}\n", + "{'input': '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"26345b28-7f75-401e-88e3-77933cb70a2e\",\"tool_name\":\"brave_search\",\"arguments\":{\"query\":\"Bill Cosby South Park episode\"}}]}', 'output': '{\"role\":\"tool\",\"call_id\":\"26345b28-7f75-401e-88e3-77933cb70a2e\",\"tool_name\":\"brave_search\",\"content\":\"{\\\\\"query\\\\\": \\\\\"Bill Cosby South Park episode\\\\\", \\\\\"top_k\\\\\": [{\\\\\"title\\\\\": \\\\\"Bill Cosby and Taylor Swift Duet - South Park Studios\\\\\", \\\\\"url\\\\\": \\\\\"https://www.southparkstudios.com/video-clips/90r7i1/south-park-bill-cosby-and-taylor-swift-duet\\\\\", \\\\\"content\\\\\": \\\\\"01:05 Bill Cosby is Here to See You South ParkS18 E10 ---------------------------------------------------- Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. 01:03 Bill Cosby and Taylor Swift Duet South ParkS18 E10 ------------------------------------------------------- The holiday special continues with Bill Cosby and Taylor Swift\\'s rendition of \\\\\\\\\\\\\"It\\'s Snowing Out There\\\\\\\\\\\\\". 01:31 #WeBelieveInYou South ParkS18 E10 -------------------------------------- With everyone watching, Kyle takes the opportunity to reach out to his brother. 01:47 Watch Your Microaggressions, Bro South ParkS19 E1 ------------------------------------------------------ Cartman\\'s plan to frame PC Principal backfires. South ParkS19 E1 -------------------------------------- After hearing that the PC people have targeted Kyle, Cartman vows to help.\\\\\", \\\\\"score\\\\\": 0.685971, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Bill Cosby is Here to See You - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/video-clips/wfot8s/south-park-bill-cosby-is-here-to-see-you\\\\\", \\\\\"content\\\\\": \\\\\"01:56 It\\'s Not About Music South ParkS18 E9 ------------------------------------------ At home, Randy sees the consequences of Lorde\\'s performance and calls the Record Producer to try and fix it. 01:24 Lorde\\'s Hologram South ParkS18 E9 -------------------------------------- The Record Producer reveals the truth about the music industry... South ParkS18 E9 --------------------------------------------- Randy catches Sharon with Tupac\\'s hologram. 01:37 I\\'ve Got Your Son, Lorde South ParkS18 E10 ----------------------------------------------- The Record Producer takes Stan and Kyle hostage. 01:05 Bill Cosby is Here to See You South ParkS18 E10 ---------------------------------------------------- Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. 01:21 Lorde Is My Dad South ParkS18 E10 -------------------------------------- After trying to confront Cartman Bra, Stan finally reveals the truth about his dad.\\\\\", \\\\\"score\\\\\": 0.6643884, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Bill Cosby (android) | South Park Character ... - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/wiki/Bill_Cosby_(android)\\\\\", \\\\\"content\\\\\": \\\\\"Bill Cosby (android) | South Park Character / Location / User talk etc | Official South Park Studios Wiki Sent back in time to destroy Eric Cartman\\'s Dawson\\'s Creek Trapper Keeper before it manifests into an omnipotent supercomputer that can destroy all humanity, \\\\\\\\\\\\\"Bill Cosby\\\\\\\\\\\\\" is really VSM471, an android or cyborg of some kind engineered by \\'hoomans\\' in the distant future. He fails in his initial missions to infiltrate South Park Elementary\\'s 4th Grade class, destroy the Trapper Keeper or Cartman himself, but with Stan Marsh and Kyle Broflovski\\'s aid, he is able to succeed in preventing his dismal future, and painfully fades from existence. South Park and all related titles, logos and characters are trademarks of Comedy Partners.\\\\\", \\\\\"score\\\\\": 0.5052006, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Trapper Keeper (South Park) - Wikipedia\\\\\", \\\\\"url\\\\\": \\\\\"https://en.wikipedia.org/wiki/Trapper_Keeper_(South_Park)\\\\\", \\\\\"content\\\\\": \\\\\"\\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" is the twelfth episode of the fourth season of the animated television series South Park, and the 60th episode of the series overall. In the episode, a man from the future wants Cartman\\'s new Trapper Keeper, while Mr. Garrison\\'s kindergarten class holds an election for class president with confusing results. It is one of the many South Park episodes that parodies a current event.[1] The main plot of the episode involving the Trapper Keeper was written before the election,[1] but the subplot is a parody of the controversy surrounding the election\\'s outcome.[2] \\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" did not originally feature the election storyline, only a subplot about Ike attending his first day of kindergarten.[3] \\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" Full episode at South Park Studios\\\\\", \\\\\"score\\\\\": 0.3839421, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Bill Cosby | South Park Archives | Fandom\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.fandom.com/wiki/Bill_Cosby\\\\\", \\\\\"content\\\\\": \\\\\"SIGN IN CHARACTERS SIGN IN Explore EXPLORE CHARACTERS SIGN IN TO EDIT Character Information For other uses, see Bill (Disambiguation). Bill Cosby is elderly, having gray hair as well as various facial wrinkles. More Information: Criminal Celebrities More Information: Movie Celebrities Minor Characters from Season Four More information: List of Minor Characters from Season Four | Season Four Community content is available under CC-BY-SA unless otherwise noted. EXPLORE PROPERTIES FOLLOW US Terms of Use Global Sitemap Local Sitemap Follow on IG\\\\\", \\\\\"score\\\\\": 0.34707275, \\\\\"raw_content\\\\\": null}]}\"}'}\n", + "{'input': ['{\"role\":\"system\",\"content\":\"You are a helpful assistant. Use search tool to answer the questions. \"}', '{\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null}', '{\"role\":\"assistant\",\"content\":\"Let me check the latest sports news.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[]}', '{\"role\":\"user\",\"content\":\"In which episode and season of South Park does Bill Cosby (BSM-471) first appear? Give me the number and title.\",\"context\":null}', '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"26345b28-7f75-401e-88e3-77933cb70a2e\",\"tool_name\":\"brave_search\",\"arguments\":{\"query\":\"Bill Cosby South Park episode\"}}]}', '{\"role\":\"tool\",\"call_id\":\"26345b28-7f75-401e-88e3-77933cb70a2e\",\"tool_name\":\"brave_search\",\"content\":\"{\\\\\"query\\\\\": \\\\\"Bill Cosby South Park episode\\\\\", \\\\\"top_k\\\\\": [{\\\\\"title\\\\\": \\\\\"Bill Cosby and Taylor Swift Duet - South Park Studios\\\\\", \\\\\"url\\\\\": \\\\\"https://www.southparkstudios.com/video-clips/90r7i1/south-park-bill-cosby-and-taylor-swift-duet\\\\\", \\\\\"content\\\\\": \\\\\"01:05 Bill Cosby is Here to See You South ParkS18 E10 ---------------------------------------------------- Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. 01:03 Bill Cosby and Taylor Swift Duet South ParkS18 E10 ------------------------------------------------------- The holiday special continues with Bill Cosby and Taylor Swift\\'s rendition of \\\\\\\\\\\\\"It\\'s Snowing Out There\\\\\\\\\\\\\". 01:31 #WeBelieveInYou South ParkS18 E10 -------------------------------------- With everyone watching, Kyle takes the opportunity to reach out to his brother. 01:47 Watch Your Microaggressions, Bro South ParkS19 E1 ------------------------------------------------------ Cartman\\'s plan to frame PC Principal backfires. South ParkS19 E1 -------------------------------------- After hearing that the PC people have targeted Kyle, Cartman vows to help.\\\\\", \\\\\"score\\\\\": 0.685971, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Bill Cosby is Here to See You - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/video-clips/wfot8s/south-park-bill-cosby-is-here-to-see-you\\\\\", \\\\\"content\\\\\": \\\\\"01:56 It\\'s Not About Music South ParkS18 E9 ------------------------------------------ At home, Randy sees the consequences of Lorde\\'s performance and calls the Record Producer to try and fix it. 01:24 Lorde\\'s Hologram South ParkS18 E9 -------------------------------------- The Record Producer reveals the truth about the music industry... South ParkS18 E9 --------------------------------------------- Randy catches Sharon with Tupac\\'s hologram. 01:37 I\\'ve Got Your Son, Lorde South ParkS18 E10 ----------------------------------------------- The Record Producer takes Stan and Kyle hostage. 01:05 Bill Cosby is Here to See You South ParkS18 E10 ---------------------------------------------------- Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. 01:21 Lorde Is My Dad South ParkS18 E10 -------------------------------------- After trying to confront Cartman Bra, Stan finally reveals the truth about his dad.\\\\\", \\\\\"score\\\\\": 0.6643884, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Bill Cosby (android) | South Park Character ... - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/wiki/Bill_Cosby_(android)\\\\\", \\\\\"content\\\\\": \\\\\"Bill Cosby (android) | South Park Character / Location / User talk etc | Official South Park Studios Wiki Sent back in time to destroy Eric Cartman\\'s Dawson\\'s Creek Trapper Keeper before it manifests into an omnipotent supercomputer that can destroy all humanity, \\\\\\\\\\\\\"Bill Cosby\\\\\\\\\\\\\" is really VSM471, an android or cyborg of some kind engineered by \\'hoomans\\' in the distant future. He fails in his initial missions to infiltrate South Park Elementary\\'s 4th Grade class, destroy the Trapper Keeper or Cartman himself, but with Stan Marsh and Kyle Broflovski\\'s aid, he is able to succeed in preventing his dismal future, and painfully fades from existence. South Park and all related titles, logos and characters are trademarks of Comedy Partners.\\\\\", \\\\\"score\\\\\": 0.5052006, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Trapper Keeper (South Park) - Wikipedia\\\\\", \\\\\"url\\\\\": \\\\\"https://en.wikipedia.org/wiki/Trapper_Keeper_(South_Park)\\\\\", \\\\\"content\\\\\": \\\\\"\\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" is the twelfth episode of the fourth season of the animated television series South Park, and the 60th episode of the series overall. In the episode, a man from the future wants Cartman\\'s new Trapper Keeper, while Mr. Garrison\\'s kindergarten class holds an election for class president with confusing results. It is one of the many South Park episodes that parodies a current event.[1] The main plot of the episode involving the Trapper Keeper was written before the election,[1] but the subplot is a parody of the controversy surrounding the election\\'s outcome.[2] \\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" did not originally feature the election storyline, only a subplot about Ike attending his first day of kindergarten.[3] \\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" Full episode at South Park Studios\\\\\", \\\\\"score\\\\\": 0.3839421, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Bill Cosby | South Park Archives | Fandom\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.fandom.com/wiki/Bill_Cosby\\\\\", \\\\\"content\\\\\": \\\\\"SIGN IN CHARACTERS SIGN IN Explore EXPLORE CHARACTERS SIGN IN TO EDIT Character Information For other uses, see Bill (Disambiguation). Bill Cosby is elderly, having gray hair as well as various facial wrinkles. More Information: Criminal Celebrities More Information: Movie Celebrities Minor Characters from Season Four More information: List of Minor Characters from Season Four | Season Four Community content is available under CC-BY-SA unless otherwise noted. EXPLORE PROPERTIES FOLLOW US Terms of Use Global Sitemap Local Sitemap Follow on IG\\\\\", \\\\\"score\\\\\": 0.34707275, \\\\\"raw_content\\\\\": null}]}\"}'], 'output': 'content: Bill Cosby (BSM-471) first appears in the episode \"Trapper Keeper\" (Season 4, Episode 12) of South Park. tool_calls: []'}\n", + "{'input': ['{\"role\":\"system\",\"content\":\"You are a helpful assistant. Use search tool to answer the questions. \"}', '{\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null}', '{\"role\":\"assistant\",\"content\":\"Let me check the latest sports news.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[]}', '{\"role\":\"user\",\"content\":\"In which episode and season of South Park does Bill Cosby (BSM-471) first appear? Give me the number and title.\",\"context\":null}', '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"26345b28-7f75-401e-88e3-77933cb70a2e\",\"tool_name\":\"brave_search\",\"arguments\":{\"query\":\"Bill Cosby South Park episode\"}}]}', '{\"role\":\"tool\",\"call_id\":\"26345b28-7f75-401e-88e3-77933cb70a2e\",\"tool_name\":\"brave_search\",\"content\":\"{\\\\\"query\\\\\": \\\\\"Bill Cosby South Park episode\\\\\", \\\\\"top_k\\\\\": [{\\\\\"title\\\\\": \\\\\"Bill Cosby and Taylor Swift Duet - South Park Studios\\\\\", \\\\\"url\\\\\": \\\\\"https://www.southparkstudios.com/video-clips/90r7i1/south-park-bill-cosby-and-taylor-swift-duet\\\\\", \\\\\"content\\\\\": \\\\\"01:05 Bill Cosby is Here to See You South ParkS18 E10 ---------------------------------------------------- Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. 01:03 Bill Cosby and Taylor Swift Duet South ParkS18 E10 ------------------------------------------------------- The holiday special continues with Bill Cosby and Taylor Swift\\'s rendition of \\\\\\\\\\\\\"It\\'s Snowing Out There\\\\\\\\\\\\\". 01:31 #WeBelieveInYou South ParkS18 E10 -------------------------------------- With everyone watching, Kyle takes the opportunity to reach out to his brother. 01:47 Watch Your Microaggressions, Bro South ParkS19 E1 ------------------------------------------------------ Cartman\\'s plan to frame PC Principal backfires. South ParkS19 E1 -------------------------------------- After hearing that the PC people have targeted Kyle, Cartman vows to help.\\\\\", \\\\\"score\\\\\": 0.685971, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Bill Cosby is Here to See You - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/video-clips/wfot8s/south-park-bill-cosby-is-here-to-see-you\\\\\", \\\\\"content\\\\\": \\\\\"01:56 It\\'s Not About Music South ParkS18 E9 ------------------------------------------ At home, Randy sees the consequences of Lorde\\'s performance and calls the Record Producer to try and fix it. 01:24 Lorde\\'s Hologram South ParkS18 E9 -------------------------------------- The Record Producer reveals the truth about the music industry... South ParkS18 E9 --------------------------------------------- Randy catches Sharon with Tupac\\'s hologram. 01:37 I\\'ve Got Your Son, Lorde South ParkS18 E10 ----------------------------------------------- The Record Producer takes Stan and Kyle hostage. 01:05 Bill Cosby is Here to See You South ParkS18 E10 ---------------------------------------------------- Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. 01:21 Lorde Is My Dad South ParkS18 E10 -------------------------------------- After trying to confront Cartman Bra, Stan finally reveals the truth about his dad.\\\\\", \\\\\"score\\\\\": 0.6643884, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Bill Cosby (android) | South Park Character ... - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/wiki/Bill_Cosby_(android)\\\\\", \\\\\"content\\\\\": \\\\\"Bill Cosby (android) | South Park Character / Location / User talk etc | Official South Park Studios Wiki Sent back in time to destroy Eric Cartman\\'s Dawson\\'s Creek Trapper Keeper before it manifests into an omnipotent supercomputer that can destroy all humanity, \\\\\\\\\\\\\"Bill Cosby\\\\\\\\\\\\\" is really VSM471, an android or cyborg of some kind engineered by \\'hoomans\\' in the distant future. He fails in his initial missions to infiltrate South Park Elementary\\'s 4th Grade class, destroy the Trapper Keeper or Cartman himself, but with Stan Marsh and Kyle Broflovski\\'s aid, he is able to succeed in preventing his dismal future, and painfully fades from existence. South Park and all related titles, logos and characters are trademarks of Comedy Partners.\\\\\", \\\\\"score\\\\\": 0.5052006, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Trapper Keeper (South Park) - Wikipedia\\\\\", \\\\\"url\\\\\": \\\\\"https://en.wikipedia.org/wiki/Trapper_Keeper_(South_Park)\\\\\", \\\\\"content\\\\\": \\\\\"\\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" is the twelfth episode of the fourth season of the animated television series South Park, and the 60th episode of the series overall. In the episode, a man from the future wants Cartman\\'s new Trapper Keeper, while Mr. Garrison\\'s kindergarten class holds an election for class president with confusing results. It is one of the many South Park episodes that parodies a current event.[1] The main plot of the episode involving the Trapper Keeper was written before the election,[1] but the subplot is a parody of the controversy surrounding the election\\'s outcome.[2] \\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" did not originally feature the election storyline, only a subplot about Ike attending his first day of kindergarten.[3] \\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" Full episode at South Park Studios\\\\\", \\\\\"score\\\\\": 0.3839421, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Bill Cosby | South Park Archives | Fandom\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.fandom.com/wiki/Bill_Cosby\\\\\", \\\\\"content\\\\\": \\\\\"SIGN IN CHARACTERS SIGN IN Explore EXPLORE CHARACTERS SIGN IN TO EDIT Character Information For other uses, see Bill (Disambiguation). Bill Cosby is elderly, having gray hair as well as various facial wrinkles. More Information: Criminal Celebrities More Information: Movie Celebrities Minor Characters from Season Four More information: List of Minor Characters from Season Four | Season Four Community content is available under CC-BY-SA unless otherwise noted. EXPLORE PROPERTIES FOLLOW US Terms of Use Global Sitemap Local Sitemap Follow on IG\\\\\", \\\\\"score\\\\\": 0.34707275, \\\\\"raw_content\\\\\": null}]}\"}', '{\"role\":\"assistant\",\"content\":\"Bill Cosby (BSM-471) first appears in the episode \\\\\"Trapper Keeper\\\\\" (Season 4, Episode 12) of South Park.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[]}', '{\"role\":\"user\",\"content\":\"What is the British-American kickboxer Andrew Tate\\'s kickboxing name?\",\"context\":null}'], 'output': \"content: tool_calls: [ToolCall(call_id='fd4cc3c6-49d0-42e4-b0af-877e72f8d6ba', tool_name=, arguments={'query': 'Andrew Tate kickboxing name'})]\"}\n", + "{'input': '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"fd4cc3c6-49d0-42e4-b0af-877e72f8d6ba\",\"tool_name\":\"brave_search\",\"arguments\":{\"query\":\"Andrew Tate kickboxing name\"}}]}', 'output': '{\"role\":\"tool\",\"call_id\":\"fd4cc3c6-49d0-42e4-b0af-877e72f8d6ba\",\"tool_name\":\"brave_search\",\"content\":\"{\\\\\"query\\\\\": \\\\\"Andrew Tate kickboxing name\\\\\", \\\\\"top_k\\\\\": [{\\\\\"title\\\\\": \\\\\"50 Facts About Andrew Tate - Facts.net\\\\\", \\\\\"url\\\\\": \\\\\"https://facts.net/andrew-tate-facts/\\\\\", \\\\\"content\\\\\": \\\\\"Full Name: Andrew Tate\\'s full name is Emory Andrew Tate III, named after his father, a celebrated chess player. Date of Birth: ... Kickboxing Start: Tate began his kickboxing career in 2005, starting his journey as a professional fighter, which would later be a significant part of his persona. First Championship:\\\\\", \\\\\"score\\\\\": 0.8967681, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Andrew Tate Age, Height, Weight, Family, Parents, Biography, Net Worth\\\\\", \\\\\"url\\\\\": \\\\\"https://biographywallah.com/andrew-tate-biography/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate Age, Height, Weight, Family, Parents, Biography, Net Worth \\\\\\\\u00bb Biography Wallah Andrew Tate Age, Height, Weight, Family, Parents, Biography, Net Worth Andrew Tate Biography NameAndrew TateReal nameEmory Andrew Tate IIIProfession \\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0Kickboxer, Commentator and BusinessmanDate of birth14 December 1986BirthplaceWashington D.C., United StatesAndrew Tate Age37 years old (as of 2024)NationalityBritish-AmericanZodiac SignSagittariusGenderMaleSchoolLocal School in Washington D.C., United StatesGirlfriend/SpouseNaghel GeorgianaSexual OrientationStraightNet worth$1000 Million Who is Andrew Tate? Andrew Tate is a British-American former professional kickboxing world champion businessman and media personality, who fought in the cruiserweight and super cruiserweight divisions. Andrew Tate Age Andrew Tate was born on 1 December 1986 and is 37 years old. Andrew Tate\\\\\\\\u2019s Net Worth What is the net worth of Andrew Tate? Where is Andrew Tate from? How old is Andrew Tate?\\\\\", \\\\\"score\\\\\": 0.80698997, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"The Life Of Andrew Tate (By Andrew Tate Himself ... - Sidekick Boxing\\\\\", \\\\\"url\\\\\": \\\\\"https://sidekickboxing.co.uk/the-life-of-andrew-king-cobra-tate/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate is a British-American former professional kickboxing world champion who fought in the cruiserweight and super cruiserweight divisions. Andrew Tate\\\\\\\\u2019s Kickboxing Career Andrew Tate in the Big Brother house Andrew Tate\\\\\\\\u2019s Kickboxing World Titles and his Sidekick boxing gloves Andrew Tate After Kickboxing Andrew Tate and his brother Tristan moved to Romania to set up their empire of businesses including trading in Bitcoin, Hustlers University, CobraTate.com, The Real World, and The War Room. From being a 4x kickboxing world champion to becoming the world\\\\\\\\u2019s most Googled man in the world with a private jet and over 33 cars, Andrew Tate\\\\\\\\u2019s life has been full of adventure.\\\\\", \\\\\"score\\\\\": 0.7817479, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"50 Facts About Andrew Tate\\\\\", \\\\\"url\\\\\": \\\\\"https://facts.net/celebrity/50-facts-about-andrew-tate/\\\\\", \\\\\"content\\\\\": \\\\\"50 Facts About Andrew Tate - Facts.net Everything Else Facts Everything Else Facts 50 Facts About Andrew Tate Known for his kickboxing prowess, internet fame, and polarizing views, Tate\\'s life is a blend of high achievements and significant legal troubles. Andrew Tate, a kickboxing champion turned internet personality, faced controversy and legal issues, showcasing the complexities of fame and the impact of social media influence on personal reputation. Andrew Tate\\'s kickboxing career is one of his most notable achievements. Andrew Tate, a former professional kickboxer turned internet personality, has made waves online with his controversial opinions and business ventures. 20 Tristan Tate Facts A Deep Dive into the Life of a Controversial Figure 47 Facts About Larenz Tate More Facts\\\\\", \\\\\"score\\\\\": 0.61834323, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Andrew Tate Kickboxing Record: Legacy of King Cobra\\\\\", \\\\\"url\\\\\": \\\\\"https://stagbite.com/andrew-tate-kickboxing-record/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate Kickboxing Record: Legacy Of King Cobra \\\\\\\\u2013 Stagbite Andrew Tate Kickboxing Record: Legacy of King Cobra Andrew Tate Kickboxing Record: Legacy of King Cobra Over the course of his career, Andrew Tate amassed an impressive kickboxing record of 76 wins and 9 losses, with 23 of those victories coming via knockout or technical knockout. Andrew Tate\\\\\\\\u2019s Kickboxing Record What is Andrew Tate\\\\\\\\u2019s kickboxing record? Andrew Tate has a kickboxing record of 76 wins and 9 losses, with 23 wins coming via knockout or technical knockout. What titles did Andrew Tate win during his kickboxing career? We talk, write, and share some of the best Internet stories on Entertainment, Culture, Travel, Food, Books along with the social media trends & viral bees.\\\\\", \\\\\"score\\\\\": 0.59796065, \\\\\"raw_content\\\\\": null}]}\"}'}\n", + "{'input': ['{\"role\":\"system\",\"content\":\"You are a helpful assistant. Use search tool to answer the questions. \"}', '{\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null}', '{\"role\":\"assistant\",\"content\":\"Let me check the latest sports news.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[]}', '{\"role\":\"user\",\"content\":\"In which episode and season of South Park does Bill Cosby (BSM-471) first appear? Give me the number and title.\",\"context\":null}', '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"26345b28-7f75-401e-88e3-77933cb70a2e\",\"tool_name\":\"brave_search\",\"arguments\":{\"query\":\"Bill Cosby South Park episode\"}}]}', '{\"role\":\"tool\",\"call_id\":\"26345b28-7f75-401e-88e3-77933cb70a2e\",\"tool_name\":\"brave_search\",\"content\":\"{\\\\\"query\\\\\": \\\\\"Bill Cosby South Park episode\\\\\", \\\\\"top_k\\\\\": [{\\\\\"title\\\\\": \\\\\"Bill Cosby and Taylor Swift Duet - South Park Studios\\\\\", \\\\\"url\\\\\": \\\\\"https://www.southparkstudios.com/video-clips/90r7i1/south-park-bill-cosby-and-taylor-swift-duet\\\\\", \\\\\"content\\\\\": \\\\\"01:05 Bill Cosby is Here to See You South ParkS18 E10 ---------------------------------------------------- Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. 01:03 Bill Cosby and Taylor Swift Duet South ParkS18 E10 ------------------------------------------------------- The holiday special continues with Bill Cosby and Taylor Swift\\'s rendition of \\\\\\\\\\\\\"It\\'s Snowing Out There\\\\\\\\\\\\\". 01:31 #WeBelieveInYou South ParkS18 E10 -------------------------------------- With everyone watching, Kyle takes the opportunity to reach out to his brother. 01:47 Watch Your Microaggressions, Bro South ParkS19 E1 ------------------------------------------------------ Cartman\\'s plan to frame PC Principal backfires. South ParkS19 E1 -------------------------------------- After hearing that the PC people have targeted Kyle, Cartman vows to help.\\\\\", \\\\\"score\\\\\": 0.685971, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Bill Cosby is Here to See You - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/video-clips/wfot8s/south-park-bill-cosby-is-here-to-see-you\\\\\", \\\\\"content\\\\\": \\\\\"01:56 It\\'s Not About Music South ParkS18 E9 ------------------------------------------ At home, Randy sees the consequences of Lorde\\'s performance and calls the Record Producer to try and fix it. 01:24 Lorde\\'s Hologram South ParkS18 E9 -------------------------------------- The Record Producer reveals the truth about the music industry... South ParkS18 E9 --------------------------------------------- Randy catches Sharon with Tupac\\'s hologram. 01:37 I\\'ve Got Your Son, Lorde South ParkS18 E10 ----------------------------------------------- The Record Producer takes Stan and Kyle hostage. 01:05 Bill Cosby is Here to See You South ParkS18 E10 ---------------------------------------------------- Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. 01:21 Lorde Is My Dad South ParkS18 E10 -------------------------------------- After trying to confront Cartman Bra, Stan finally reveals the truth about his dad.\\\\\", \\\\\"score\\\\\": 0.6643884, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Bill Cosby (android) | South Park Character ... - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/wiki/Bill_Cosby_(android)\\\\\", \\\\\"content\\\\\": \\\\\"Bill Cosby (android) | South Park Character / Location / User talk etc | Official South Park Studios Wiki Sent back in time to destroy Eric Cartman\\'s Dawson\\'s Creek Trapper Keeper before it manifests into an omnipotent supercomputer that can destroy all humanity, \\\\\\\\\\\\\"Bill Cosby\\\\\\\\\\\\\" is really VSM471, an android or cyborg of some kind engineered by \\'hoomans\\' in the distant future. He fails in his initial missions to infiltrate South Park Elementary\\'s 4th Grade class, destroy the Trapper Keeper or Cartman himself, but with Stan Marsh and Kyle Broflovski\\'s aid, he is able to succeed in preventing his dismal future, and painfully fades from existence. South Park and all related titles, logos and characters are trademarks of Comedy Partners.\\\\\", \\\\\"score\\\\\": 0.5052006, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Trapper Keeper (South Park) - Wikipedia\\\\\", \\\\\"url\\\\\": \\\\\"https://en.wikipedia.org/wiki/Trapper_Keeper_(South_Park)\\\\\", \\\\\"content\\\\\": \\\\\"\\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" is the twelfth episode of the fourth season of the animated television series South Park, and the 60th episode of the series overall. In the episode, a man from the future wants Cartman\\'s new Trapper Keeper, while Mr. Garrison\\'s kindergarten class holds an election for class president with confusing results. It is one of the many South Park episodes that parodies a current event.[1] The main plot of the episode involving the Trapper Keeper was written before the election,[1] but the subplot is a parody of the controversy surrounding the election\\'s outcome.[2] \\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" did not originally feature the election storyline, only a subplot about Ike attending his first day of kindergarten.[3] \\\\\\\\\\\\\"Trapper Keeper\\\\\\\\\\\\\" Full episode at South Park Studios\\\\\", \\\\\"score\\\\\": 0.3839421, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Bill Cosby | South Park Archives | Fandom\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.fandom.com/wiki/Bill_Cosby\\\\\", \\\\\"content\\\\\": \\\\\"SIGN IN CHARACTERS SIGN IN Explore EXPLORE CHARACTERS SIGN IN TO EDIT Character Information For other uses, see Bill (Disambiguation). Bill Cosby is elderly, having gray hair as well as various facial wrinkles. More Information: Criminal Celebrities More Information: Movie Celebrities Minor Characters from Season Four More information: List of Minor Characters from Season Four | Season Four Community content is available under CC-BY-SA unless otherwise noted. EXPLORE PROPERTIES FOLLOW US Terms of Use Global Sitemap Local Sitemap Follow on IG\\\\\", \\\\\"score\\\\\": 0.34707275, \\\\\"raw_content\\\\\": null}]}\"}', '{\"role\":\"assistant\",\"content\":\"Bill Cosby (BSM-471) first appears in the episode \\\\\"Trapper Keeper\\\\\" (Season 4, Episode 12) of South Park.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[]}', '{\"role\":\"user\",\"content\":\"What is the British-American kickboxer Andrew Tate\\'s kickboxing name?\",\"context\":null}', '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"fd4cc3c6-49d0-42e4-b0af-877e72f8d6ba\",\"tool_name\":\"brave_search\",\"arguments\":{\"query\":\"Andrew Tate kickboxing name\"}}]}', '{\"role\":\"tool\",\"call_id\":\"fd4cc3c6-49d0-42e4-b0af-877e72f8d6ba\",\"tool_name\":\"brave_search\",\"content\":\"{\\\\\"query\\\\\": \\\\\"Andrew Tate kickboxing name\\\\\", \\\\\"top_k\\\\\": [{\\\\\"title\\\\\": \\\\\"50 Facts About Andrew Tate - Facts.net\\\\\", \\\\\"url\\\\\": \\\\\"https://facts.net/andrew-tate-facts/\\\\\", \\\\\"content\\\\\": \\\\\"Full Name: Andrew Tate\\'s full name is Emory Andrew Tate III, named after his father, a celebrated chess player. Date of Birth: ... Kickboxing Start: Tate began his kickboxing career in 2005, starting his journey as a professional fighter, which would later be a significant part of his persona. First Championship:\\\\\", \\\\\"score\\\\\": 0.8967681, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Andrew Tate Age, Height, Weight, Family, Parents, Biography, Net Worth\\\\\", \\\\\"url\\\\\": \\\\\"https://biographywallah.com/andrew-tate-biography/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate Age, Height, Weight, Family, Parents, Biography, Net Worth \\\\\\\\u00bb Biography Wallah Andrew Tate Age, Height, Weight, Family, Parents, Biography, Net Worth Andrew Tate Biography NameAndrew TateReal nameEmory Andrew Tate IIIProfession \\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0\\\\\\\\u00a0Kickboxer, Commentator and BusinessmanDate of birth14 December 1986BirthplaceWashington D.C., United StatesAndrew Tate Age37 years old (as of 2024)NationalityBritish-AmericanZodiac SignSagittariusGenderMaleSchoolLocal School in Washington D.C., United StatesGirlfriend/SpouseNaghel GeorgianaSexual OrientationStraightNet worth$1000 Million Who is Andrew Tate? Andrew Tate is a British-American former professional kickboxing world champion businessman and media personality, who fought in the cruiserweight and super cruiserweight divisions. Andrew Tate Age Andrew Tate was born on 1 December 1986 and is 37 years old. Andrew Tate\\\\\\\\u2019s Net Worth What is the net worth of Andrew Tate? Where is Andrew Tate from? How old is Andrew Tate?\\\\\", \\\\\"score\\\\\": 0.80698997, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"The Life Of Andrew Tate (By Andrew Tate Himself ... - Sidekick Boxing\\\\\", \\\\\"url\\\\\": \\\\\"https://sidekickboxing.co.uk/the-life-of-andrew-king-cobra-tate/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate is a British-American former professional kickboxing world champion who fought in the cruiserweight and super cruiserweight divisions. Andrew Tate\\\\\\\\u2019s Kickboxing Career Andrew Tate in the Big Brother house Andrew Tate\\\\\\\\u2019s Kickboxing World Titles and his Sidekick boxing gloves Andrew Tate After Kickboxing Andrew Tate and his brother Tristan moved to Romania to set up their empire of businesses including trading in Bitcoin, Hustlers University, CobraTate.com, The Real World, and The War Room. From being a 4x kickboxing world champion to becoming the world\\\\\\\\u2019s most Googled man in the world with a private jet and over 33 cars, Andrew Tate\\\\\\\\u2019s life has been full of adventure.\\\\\", \\\\\"score\\\\\": 0.7817479, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"50 Facts About Andrew Tate\\\\\", \\\\\"url\\\\\": \\\\\"https://facts.net/celebrity/50-facts-about-andrew-tate/\\\\\", \\\\\"content\\\\\": \\\\\"50 Facts About Andrew Tate - Facts.net Everything Else Facts Everything Else Facts 50 Facts About Andrew Tate Known for his kickboxing prowess, internet fame, and polarizing views, Tate\\'s life is a blend of high achievements and significant legal troubles. Andrew Tate, a kickboxing champion turned internet personality, faced controversy and legal issues, showcasing the complexities of fame and the impact of social media influence on personal reputation. Andrew Tate\\'s kickboxing career is one of his most notable achievements. Andrew Tate, a former professional kickboxer turned internet personality, has made waves online with his controversial opinions and business ventures. 20 Tristan Tate Facts A Deep Dive into the Life of a Controversial Figure 47 Facts About Larenz Tate More Facts\\\\\", \\\\\"score\\\\\": 0.61834323, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Andrew Tate Kickboxing Record: Legacy of King Cobra\\\\\", \\\\\"url\\\\\": \\\\\"https://stagbite.com/andrew-tate-kickboxing-record/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate Kickboxing Record: Legacy Of King Cobra \\\\\\\\u2013 Stagbite Andrew Tate Kickboxing Record: Legacy of King Cobra Andrew Tate Kickboxing Record: Legacy of King Cobra Over the course of his career, Andrew Tate amassed an impressive kickboxing record of 76 wins and 9 losses, with 23 of those victories coming via knockout or technical knockout. Andrew Tate\\\\\\\\u2019s Kickboxing Record What is Andrew Tate\\\\\\\\u2019s kickboxing record? Andrew Tate has a kickboxing record of 76 wins and 9 losses, with 23 wins coming via knockout or technical knockout. What titles did Andrew Tate win during his kickboxing career? We talk, write, and share some of the best Internet stories on Entertainment, Culture, Travel, Food, Books along with the social media trends & viral bees.\\\\\", \\\\\"score\\\\\": 0.59796065, \\\\\"raw_content\\\\\": null}]}\"}'], 'output': 'content: Andrew Tate\\'s kickboxing name is \"King Cobra.\" tool_calls: []'}\n" + ] + }, { "data": { "text/html": [ @@ -2318,12 +3593,12 @@ "│ },\n", "│ {\n", "│ │ 'input_query': '{\"role\":\"user\",\"content\":\"In which episode and season of South Park does Bill Cosby (BSM-471) first appear? Give me the number and title.\",\"context\":null}',\n", - "│ │ 'generated_answer': \"content: tool_calls: [ToolCall(call_id='19bd3554-e670-4856-89d0-c63f5b016245', tool_name='bravy_search', arguments={'query': 'Bill Cosby South Park episode'})]\",\n", - "│ │ 'expected_answer': 'brave_search'\n", - "│ },\n", - "│ {\n", - "│ │ 'input_query': '{\"role\":\"user\",\"content\":\"What is the British-American kickboxer Andrew Tate\\'s kickboxing name?\",\"context\":null}',\n", - "│ │ 'generated_answer': \"content: tool_calls: [ToolCall(call_id='526045a7-5f51-40fb-ba97-5ad29610e511', tool_name=<BuiltinTool.brave_search: 'brave_search'>, arguments={'query': 'Andrew Tate kickboxing name'})]\",\n", + "│ │ 'generated_answer': \"content: tool_calls: [ToolCall(call_id='26345b28-7f75-401e-88e3-77933cb70a2e', tool_name=<BuiltinTool.brave_search: 'brave_search'>, arguments={'query': 'Bill Cosby South Park episode'})]\",\n", + "│ │ 'expected_answer': 'brave_search'\n", + "│ },\n", + "│ {\n", + "│ │ 'input_query': '{\"role\":\"user\",\"content\":\"What is the British-American kickboxer Andrew Tate\\'s kickboxing name?\",\"context\":null}',\n", + "│ │ 'generated_answer': \"content: tool_calls: [ToolCall(call_id='fd4cc3c6-49d0-42e4-b0af-877e72f8d6ba', tool_name=<BuiltinTool.brave_search: 'brave_search'>, arguments={'query': 'Andrew Tate kickboxing name'})]\",\n", "│ │ 'expected_answer': 'brave_search'\n", "│ }\n", "]\n", @@ -2338,12 +3613,12 @@ "\u001b[2;32m│ \u001b[0m\u001b[1m}\u001b[0m,\n", "\u001b[2;32m│ \u001b[0m\u001b[1m{\u001b[0m\n", "\u001b[2;32m│ │ \u001b[0m\u001b[32m'input_query'\u001b[0m: \u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"user\",\"content\":\"In which episode and season of South Park does Bill Cosby \u001b[0m\u001b[32m(\u001b[0m\u001b[32mBSM-471\u001b[0m\u001b[32m)\u001b[0m\u001b[32m first appear? Give me the number and title.\",\"context\":null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m\"content: tool_calls: \u001b[0m\u001b[32m[\u001b[0m\u001b[32mToolCall\u001b[0m\u001b[32m(\u001b[0m\u001b[32mcall_id\u001b[0m\u001b[32m='19bd3554-e670-4856-89d0-c63f5b016245', \u001b[0m\u001b[32mtool_name\u001b[0m\u001b[32m='bravy_search', \u001b[0m\u001b[32marguments\u001b[0m\u001b[32m=\u001b[0m\u001b[32m{\u001b[0m\u001b[32m'query': 'Bill Cosby South Park episode'\u001b[0m\u001b[32m}\u001b[0m\u001b[32m)\u001b[0m\u001b[32m]\u001b[0m\u001b[32m\"\u001b[0m,\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'expected_answer'\u001b[0m: \u001b[32m'brave_search'\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[1m}\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[1m{\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'input_query'\u001b[0m: \u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"user\",\"content\":\"What is the British-American kickboxer Andrew Tate\\'s kickboxing name?\",\"context\":null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m\"content: tool_calls: \u001b[0m\u001b[32m[\u001b[0m\u001b[32mToolCall\u001b[0m\u001b[32m(\u001b[0m\u001b[32mcall_id\u001b[0m\u001b[32m='526045a7-5f51-40fb-ba97-5ad29610e511', \u001b[0m\u001b[32mtool_name\u001b[0m\u001b[32m=\u001b[0m\u001b[32m<\u001b[0m\u001b[32mBuiltinTool.brave_search:\u001b[0m\u001b[32m 'brave_search'\u001b[0m\u001b[32m>\u001b[0m\u001b[32m, \u001b[0m\u001b[32marguments\u001b[0m\u001b[32m=\u001b[0m\u001b[32m{\u001b[0m\u001b[32m'query': 'Andrew Tate kickboxing name'\u001b[0m\u001b[32m}\u001b[0m\u001b[32m)\u001b[0m\u001b[32m]\u001b[0m\u001b[32m\"\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m\"content: tool_calls: \u001b[0m\u001b[32m[\u001b[0m\u001b[32mToolCall\u001b[0m\u001b[32m(\u001b[0m\u001b[32mcall_id\u001b[0m\u001b[32m='26345b28-7f75-401e-88e3-77933cb70a2e', \u001b[0m\u001b[32mtool_name\u001b[0m\u001b[32m=\u001b[0m\u001b[32m<\u001b[0m\u001b[32mBuiltinTool.brave_search:\u001b[0m\u001b[32m 'brave_search'>, \u001b[0m\u001b[32marguments\u001b[0m\u001b[32m=\u001b[0m\u001b[32m{\u001b[0m\u001b[32m'query': 'Bill Cosby South Park episode'\u001b[0m\u001b[32m}\u001b[0m\u001b[32m)\u001b[0m\u001b[32m]\u001b[0m\u001b[32m\"\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[32m'expected_answer'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'brave_search'\u001b[0m\n", + "\u001b[2;32m│ \u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ \u001b[0m\u001b[1;39m{\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[32m'input_query'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"user\",\"content\":\"What is the British-American kickboxer Andrew Tate\\'s kickboxing name?\",\"context\":null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[32m'generated_answer'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m\"content: tool_calls: \u001b[0m\u001b[32m[\u001b[0m\u001b[32mToolCall\u001b[0m\u001b[32m(\u001b[0m\u001b[32mcall_id\u001b[0m\u001b[32m='fd4cc3c6-49d0-42e4-b0af-877e72f8d6ba', \u001b[0m\u001b[32mtool_name\u001b[0m\u001b[32m=\u001b[0m\u001b[32m, \u001b[0m\u001b[32marguments\u001b[0m\u001b[32m=\u001b[0m\u001b[32m{\u001b[0m\u001b[32m'query': 'Andrew Tate kickboxing name'\u001b[0m\u001b[32m}\u001b[0m\u001b[32m)\u001b[0m\u001b[32m]\u001b[0m\u001b[32m\"\u001b[0m,\n", "\u001b[2;32m│ │ \u001b[0m\u001b[32m'expected_answer'\u001b[0m: \u001b[32m'brave_search'\u001b[0m\n", "\u001b[2;32m│ \u001b[0m\u001b[1m}\u001b[0m\n", "\u001b[1m]\u001b[0m\n" @@ -2358,8 +3633,8 @@ "
ScoringScoreResponse(\n",
               "│   results={\n",
               "│   │   'basic::subset_of': ScoringResult(\n",
-              "│   │   │   aggregated_results={'accuracy': {'accuracy': 0.3333333333333333, 'num_correct': 1.0, 'num_total': 3}},\n",
-              "│   │   │   score_rows=[{'score': 0.0}, {'score': 0.0}, {'score': 1.0}]\n",
+              "│   │   │   aggregated_results={'accuracy': {'accuracy': 0.6666666666666666, 'num_correct': 2.0, 'num_total': 3}},\n",
+              "│   │   │   score_rows=[{'score': 0.0}, {'score': 1.0}, {'score': 1.0}]\n",
               "│   │   )\n",
               "│   }\n",
               ")\n",
@@ -2369,8 +3644,8 @@
               "\u001b[1;35mScoringScoreResponse\u001b[0m\u001b[1m(\u001b[0m\n",
               "\u001b[2;32m│   \u001b[0m\u001b[33mresults\u001b[0m=\u001b[1m{\u001b[0m\n",
               "\u001b[2;32m│   │   \u001b[0m\u001b[32m'basic::subset_of'\u001b[0m: \u001b[1;35mScoringResult\u001b[0m\u001b[1m(\u001b[0m\n",
-              "\u001b[2;32m│   │   │   \u001b[0m\u001b[33maggregated_results\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'accuracy'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'accuracy'\u001b[0m: \u001b[1;36m0.3333333333333333\u001b[0m, \u001b[32m'num_correct'\u001b[0m: \u001b[1;36m1.0\u001b[0m, \u001b[32m'num_total'\u001b[0m: \u001b[1;36m3\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n",
-              "\u001b[2;32m│   │   │   \u001b[0m\u001b[33mscore_rows\u001b[0m=\u001b[1m[\u001b[0m\u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m, \u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m, \u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[1;36m1.0\u001b[0m\u001b[1m}\u001b[0m\u001b[1m]\u001b[0m\n",
+              "\u001b[2;32m│   │   │   \u001b[0m\u001b[33maggregated_results\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'accuracy'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'accuracy'\u001b[0m: \u001b[1;36m0.6666666666666666\u001b[0m, \u001b[32m'num_correct'\u001b[0m: \u001b[1;36m2.0\u001b[0m, \u001b[32m'num_total'\u001b[0m: \u001b[1;36m3\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n",
+              "\u001b[2;32m│   │   │   \u001b[0m\u001b[33mscore_rows\u001b[0m=\u001b[1m[\u001b[0m\u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m, \u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[1;36m1.0\u001b[0m\u001b[1m}\u001b[0m, \u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[1;36m1.0\u001b[0m\u001b[1m}\u001b[0m\u001b[1m]\u001b[0m\n",
               "\u001b[2;32m│   │   \u001b[0m\u001b[1m)\u001b[0m\n",
               "\u001b[2;32m│   \u001b[0m\u001b[1m}\u001b[0m\n",
               "\u001b[1m)\u001b[0m\n"
@@ -2381,6 +3656,7 @@
         }
       ],
       "source": [
+        "# NBVAL_SKIP\n",
         "# post-process telemetry spance and prepare data for eval\n",
         "# in this case, we want to assert that all user prompts is followed by a tool call\n",
         "import ast\n",
@@ -2389,23 +3665,26 @@
         "eval_rows = []\n",
         "\n",
         "for log in agent_logs:\n",
-        "  last_msg = log['input'][-1]\n",
-        "  if \"\\\"role\\\":\\\"user\\\"\" in last_msg:\n",
-        "    eval_rows.append(\n",
-        "        {\n",
-        "            \"input_query\": last_msg,\n",
-        "            \"generated_answer\": log[\"output\"],\n",
-        "            # check if generated_answer uses tools brave_search\n",
-        "            \"expected_answer\": \"brave_search\",\n",
-        "        },\n",
-        "    )\n",
+        "    print(log)\n",
+        "    last_msg = log[\"input\"][-1]\n",
+        "    if '\"role\":\"user\"' in last_msg:\n",
+        "        eval_rows.append(\n",
+        "            {\n",
+        "                \"input_query\": last_msg,\n",
+        "                \"generated_answer\": log[\"output\"],\n",
+        "                # check if generated_answer uses tools brave_search\n",
+        "                \"expected_answer\": \"brave_search\",\n",
+        "            },\n",
+        "        )\n",
         "\n",
         "pprint(eval_rows)\n",
         "scoring_params = {\n",
         "    \"basic::subset_of\": None,\n",
         "}\n",
-        "scoring_response = client.scoring.score(input_rows=eval_rows, scoring_functions=scoring_params)\n",
-        "pprint(scoring_response)"
+        "scoring_response = client.scoring.score(\n",
+        "    input_rows=eval_rows, scoring_functions=scoring_params\n",
+        ")\n",
+        "pprint(scoring_response)\n"
       ]
     },
     {
@@ -2428,10 +3707,10 @@
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/",
-          "height": 298
+          "height": 304
         },
         "id": "xG4Y84VQBb0g",
-        "outputId": "f61cebdf-f614-440c-d170-f1e873b542ef"
+        "outputId": "cf7dcecc-a81d-4c60-af5e-b36b8fe85c69"
       },
       "outputs": [
         {
@@ -2444,12 +3723,12 @@
               "│   │   │   score_rows=[\n",
               "│   │   │   │   {\n",
               "│   │   │   │   │   'score': 'B',\n",
-              "│   │   │   │   │   'judge_feedback': 'Answer: B, Explanation: The GENERATED_RESPONSE is a superset of the EXPECTED_RESPONSE and is fully consistent with it. The GENERATED_RESPONSE provides more detailed information about the top 5 topics related to LoRA, while the EXPECTED_RESPONSE only mentions \"LoRA\". The GENERATED_RESPONSE expands on the topic, but does not conflict with the EXPECTED_RESPONSE.'\n",
+              "│   │   │   │   │   'judge_feedback': 'Answer: B, Explanation: The GENERATED_RESPONSE is a superset of the EXPECTED_RESPONSE and is fully consistent with it. The EXPECTED_RESPONSE only mentions \"LoRA\", which is present in all the points of the GENERATED_RESPONSE. The GENERATED_RESPONSE provides more details and specific topics related to LoRA, but it does not contradict the EXPECTED_RESPONSE.'\n",
               "│   │   │   │   }\n",
               "│   │   │   ]\n",
               "│   │   ),\n",
               "│   │   'basic::subset_of': ScoringResult(\n",
-              "│   │   │   aggregated_results={'accuracy': 1.0, 'num_correct': 1.0, 'num_total': 1.0},\n",
+              "│   │   │   aggregated_results={'accuracy': {'accuracy': 1.0, 'num_correct': 1.0, 'num_total': 1}},\n",
               "│   │   │   score_rows=[{'score': 1.0}]\n",
               "│   │   )\n",
               "│   }\n",
@@ -2464,12 +3743,12 @@
               "\u001b[2;32m│   │   │   \u001b[0m\u001b[33mscore_rows\u001b[0m=\u001b[1m[\u001b[0m\n",
               "\u001b[2;32m│   │   │   │   \u001b[0m\u001b[1m{\u001b[0m\n",
               "\u001b[2;32m│   │   │   │   │   \u001b[0m\u001b[32m'score'\u001b[0m: \u001b[32m'B'\u001b[0m,\n",
-              "\u001b[2;32m│   │   │   │   │   \u001b[0m\u001b[32m'judge_feedback'\u001b[0m: \u001b[32m'Answer: B, Explanation: The GENERATED_RESPONSE is a superset of the EXPECTED_RESPONSE and is fully consistent with it. The GENERATED_RESPONSE provides more detailed information about the top 5 topics related to LoRA, while the EXPECTED_RESPONSE only mentions \"LoRA\". The GENERATED_RESPONSE expands on the topic, but does not conflict with the EXPECTED_RESPONSE.'\u001b[0m\n",
+              "\u001b[2;32m│   │   │   │   │   \u001b[0m\u001b[32m'judge_feedback'\u001b[0m: \u001b[32m'Answer: B, Explanation: The GENERATED_RESPONSE is a superset of the EXPECTED_RESPONSE and is fully consistent with it. The EXPECTED_RESPONSE only mentions \"LoRA\", which is present in all the points of the GENERATED_RESPONSE. The GENERATED_RESPONSE provides more details and specific topics related to LoRA, but it does not contradict the EXPECTED_RESPONSE.'\u001b[0m\n",
               "\u001b[2;32m│   │   │   │   \u001b[0m\u001b[1m}\u001b[0m\n",
               "\u001b[2;32m│   │   │   \u001b[0m\u001b[1m]\u001b[0m\n",
               "\u001b[2;32m│   │   \u001b[0m\u001b[1m)\u001b[0m,\n",
               "\u001b[2;32m│   │   \u001b[0m\u001b[32m'basic::subset_of'\u001b[0m: \u001b[1;35mScoringResult\u001b[0m\u001b[1m(\u001b[0m\n",
-              "\u001b[2;32m│   │   │   \u001b[0m\u001b[33maggregated_results\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'accuracy'\u001b[0m: \u001b[1;36m1.0\u001b[0m, \u001b[32m'num_correct'\u001b[0m: \u001b[1;36m1.0\u001b[0m, \u001b[32m'num_total'\u001b[0m: \u001b[1;36m1.0\u001b[0m\u001b[1m}\u001b[0m,\n",
+              "\u001b[2;32m│   │   │   \u001b[0m\u001b[33maggregated_results\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'accuracy'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'accuracy'\u001b[0m: \u001b[1;36m1.0\u001b[0m, \u001b[32m'num_correct'\u001b[0m: \u001b[1;36m1.0\u001b[0m, \u001b[32m'num_total'\u001b[0m: \u001b[1;36m1\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n",
               "\u001b[2;32m│   │   │   \u001b[0m\u001b[33mscore_rows\u001b[0m=\u001b[1m[\u001b[0m\u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[1;36m1.0\u001b[0m\u001b[1m}\u001b[0m\u001b[1m]\u001b[0m\n",
               "\u001b[2;32m│   │   \u001b[0m\u001b[1m)\u001b[0m\n",
               "\u001b[2;32m│   \u001b[0m\u001b[1m}\u001b[0m\n",
@@ -2481,6 +3760,7 @@
         }
       ],
       "source": [
+        "# NBVAL_SKIP\n",
         "import rich\n",
         "from rich.pretty import pprint\n",
         "\n",
@@ -2506,7 +3786,9 @@
         "EXPECTED_RESPONSE: {expected_answer}\n",
         "\"\"\"\n",
         "\n",
-        "input_query = \"What are the top 5 topics that were explained? Only list succinct bullet points.\"\n",
+        "input_query = (\n",
+        "    \"What are the top 5 topics that were explained? Only list succinct bullet points.\"\n",
+        ")\n",
         "generated_answer = \"\"\"\n",
         "Here are the top 5 topics that were explained in the documentation for Torchtune:\n",
         "\n",
@@ -2537,25 +3819,14 @@
         "}\n",
         "\n",
         "response = client.scoring.score(input_rows=rows, scoring_functions=scoring_params)\n",
-        "pprint(response)"
+        "pprint(response)\n"
       ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "id": "rKtGo_v98UA2",
-      "metadata": {
-        "id": "rKtGo_v98UA2"
-      },
-      "outputs": [],
-      "source": []
     }
   ],
   "metadata": {
+    "accelerator": "GPU",
     "colab": {
-      "collapsed_sections": [
-        "_JueJAKyJR5m"
-      ],
+      "gpuType": "T4",
       "provenance": []
     },
     "kernelspec": {
@@ -2576,7 +3847,7 @@
     },
     "widgets": {
       "application/vnd.jupyter.widget-state+json": {
-        "0243626d7ef44ef2b90e8fed5c13183d": {
+        "028e291ee53947bbbbc4bfb68c695f5f": {
           "model_module": "@jupyter-widgets/controls",
           "model_module_version": "1.5.0",
           "model_name": "DescriptionStyleModel",
@@ -2591,7 +3862,59 @@
             "description_width": ""
           }
         },
-        "044d6d8dda1c4935b1752a9c71c6ee4a": {
+        "02baf670942347d69c290452de8641e4": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "03402ad03418435ca7a550e3246cd300": {
           "model_module": "@jupyter-widgets/controls",
           "model_module_version": "1.5.0",
           "model_name": "FloatProgressModel",
@@ -2607,89 +3930,15 @@
             "bar_style": "success",
             "description": "",
             "description_tooltip": null,
-            "layout": "IPY_MODEL_63f34c3d43bb4fdd9faeb6161fd77285",
+            "layout": "IPY_MODEL_9df914248c214597bed7d7980c7a0afe",
             "max": 1,
             "min": 0,
             "orientation": "horizontal",
-            "style": "IPY_MODEL_5cb841b49eaa429e8616ec4b78f501e9",
+            "style": "IPY_MODEL_4709067f3f554b93b3ef35e3f58cbf85",
             "value": 1
           }
         },
-        "0640b57408644741970dd958ca0e21e6": {
-          "model_module": "@jupyter-widgets/controls",
-          "model_module_version": "1.5.0",
-          "model_name": "HBoxModel",
-          "state": {
-            "_dom_classes": [],
-            "_model_module": "@jupyter-widgets/controls",
-            "_model_module_version": "1.5.0",
-            "_model_name": "HBoxModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/controls",
-            "_view_module_version": "1.5.0",
-            "_view_name": "HBoxView",
-            "box_style": "",
-            "children": [
-              "IPY_MODEL_6259ffc3ef674df985fd3fa4334f9c8e",
-              "IPY_MODEL_3d0376d2e574410eb4ef963d51cac0a6",
-              "IPY_MODEL_b66984cc5de541a5801a1e6e54d40daf"
-            ],
-            "layout": "IPY_MODEL_92135b9cb201475681ee0886887c84a8"
-          }
-        },
-        "116139bfe7a44f969a2c97490c224d31": {
-          "model_module": "@jupyter-widgets/controls",
-          "model_module_version": "1.5.0",
-          "model_name": "HTMLModel",
-          "state": {
-            "_dom_classes": [],
-            "_model_module": "@jupyter-widgets/controls",
-            "_model_module_version": "1.5.0",
-            "_model_name": "HTMLModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/controls",
-            "_view_module_version": "1.5.0",
-            "_view_name": "HTMLView",
-            "description": "",
-            "description_tooltip": null,
-            "layout": "IPY_MODEL_ab1f339cba094c918fc5507f8361de5c",
-            "placeholder": "​",
-            "style": "IPY_MODEL_a6a1eb412f204578b80e5b6717c1e3a5",
-            "value": " 1/1 [00:01<00:00,  1.27s/it]"
-          }
-        },
-        "118b359b83304ae59fad57e28f621645": {
-          "model_module": "@jupyter-widgets/controls",
-          "model_module_version": "1.5.0",
-          "model_name": "ProgressStyleModel",
-          "state": {
-            "_model_module": "@jupyter-widgets/controls",
-            "_model_module_version": "1.5.0",
-            "_model_name": "ProgressStyleModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/base",
-            "_view_module_version": "1.2.0",
-            "_view_name": "StyleView",
-            "bar_color": null,
-            "description_width": ""
-          }
-        },
-        "15d3ff07f1c54e58b51d452caca01209": {
-          "model_module": "@jupyter-widgets/controls",
-          "model_module_version": "1.5.0",
-          "model_name": "DescriptionStyleModel",
-          "state": {
-            "_model_module": "@jupyter-widgets/controls",
-            "_model_module_version": "1.5.0",
-            "_model_name": "DescriptionStyleModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/base",
-            "_view_module_version": "1.2.0",
-            "_view_name": "StyleView",
-            "description_width": ""
-          }
-        },
-        "17603dd7fedf4798a74533fbfd5bb421": {
+        "03bbebd659e64b5d9c29a73570c34854": {
           "model_module": "@jupyter-widgets/base",
           "model_module_version": "1.2.0",
           "model_name": "LayoutModel",
@@ -2741,304 +3990,7 @@
             "width": null
           }
         },
-        "186682be50c148c0826fa7c314087562": {
-          "model_module": "@jupyter-widgets/controls",
-          "model_module_version": "1.5.0",
-          "model_name": "HTMLModel",
-          "state": {
-            "_dom_classes": [],
-            "_model_module": "@jupyter-widgets/controls",
-            "_model_module_version": "1.5.0",
-            "_model_name": "HTMLModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/controls",
-            "_view_module_version": "1.5.0",
-            "_view_name": "HTMLView",
-            "description": "",
-            "description_tooltip": null,
-            "layout": "IPY_MODEL_1f427d4273e04e19b1bdb13388736c01",
-            "placeholder": "​",
-            "style": "IPY_MODEL_38897429b7cf4077aea3a981593ca866",
-            "value": " 1/1 [00:00<00:00, 15.09it/s]"
-          }
-        },
-        "1f427d4273e04e19b1bdb13388736c01": {
-          "model_module": "@jupyter-widgets/base",
-          "model_module_version": "1.2.0",
-          "model_name": "LayoutModel",
-          "state": {
-            "_model_module": "@jupyter-widgets/base",
-            "_model_module_version": "1.2.0",
-            "_model_name": "LayoutModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/base",
-            "_view_module_version": "1.2.0",
-            "_view_name": "LayoutView",
-            "align_content": null,
-            "align_items": null,
-            "align_self": null,
-            "border": null,
-            "bottom": null,
-            "display": null,
-            "flex": null,
-            "flex_flow": null,
-            "grid_area": null,
-            "grid_auto_columns": null,
-            "grid_auto_flow": null,
-            "grid_auto_rows": null,
-            "grid_column": null,
-            "grid_gap": null,
-            "grid_row": null,
-            "grid_template_areas": null,
-            "grid_template_columns": null,
-            "grid_template_rows": null,
-            "height": null,
-            "justify_content": null,
-            "justify_items": null,
-            "left": null,
-            "margin": null,
-            "max_height": null,
-            "max_width": null,
-            "min_height": null,
-            "min_width": null,
-            "object_fit": null,
-            "object_position": null,
-            "order": null,
-            "overflow": null,
-            "overflow_x": null,
-            "overflow_y": null,
-            "padding": null,
-            "right": null,
-            "top": null,
-            "visibility": null,
-            "width": null
-          }
-        },
-        "2082554eed6644a996f0e31545789e08": {
-          "model_module": "@jupyter-widgets/controls",
-          "model_module_version": "1.5.0",
-          "model_name": "HBoxModel",
-          "state": {
-            "_dom_classes": [],
-            "_model_module": "@jupyter-widgets/controls",
-            "_model_module_version": "1.5.0",
-            "_model_name": "HBoxModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/controls",
-            "_view_module_version": "1.5.0",
-            "_view_name": "HBoxView",
-            "box_style": "",
-            "children": [
-              "IPY_MODEL_a0be415018644c3cac098ab9b19c2391",
-              "IPY_MODEL_6ede3649e8c24015b3ca77490568bfcd",
-              "IPY_MODEL_116139bfe7a44f969a2c97490c224d31"
-            ],
-            "layout": "IPY_MODEL_243d13828d854880a6adb861ea867734"
-          }
-        },
-        "2100363a158b4488a58620983aa5bdd4": {
-          "model_module": "@jupyter-widgets/controls",
-          "model_module_version": "1.5.0",
-          "model_name": "DescriptionStyleModel",
-          "state": {
-            "_model_module": "@jupyter-widgets/controls",
-            "_model_module_version": "1.5.0",
-            "_model_name": "DescriptionStyleModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/base",
-            "_view_module_version": "1.2.0",
-            "_view_name": "StyleView",
-            "description_width": ""
-          }
-        },
-        "243d13828d854880a6adb861ea867734": {
-          "model_module": "@jupyter-widgets/base",
-          "model_module_version": "1.2.0",
-          "model_name": "LayoutModel",
-          "state": {
-            "_model_module": "@jupyter-widgets/base",
-            "_model_module_version": "1.2.0",
-            "_model_name": "LayoutModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/base",
-            "_view_module_version": "1.2.0",
-            "_view_name": "LayoutView",
-            "align_content": null,
-            "align_items": null,
-            "align_self": null,
-            "border": null,
-            "bottom": null,
-            "display": null,
-            "flex": null,
-            "flex_flow": null,
-            "grid_area": null,
-            "grid_auto_columns": null,
-            "grid_auto_flow": null,
-            "grid_auto_rows": null,
-            "grid_column": null,
-            "grid_gap": null,
-            "grid_row": null,
-            "grid_template_areas": null,
-            "grid_template_columns": null,
-            "grid_template_rows": null,
-            "height": null,
-            "justify_content": null,
-            "justify_items": null,
-            "left": null,
-            "margin": null,
-            "max_height": null,
-            "max_width": null,
-            "min_height": null,
-            "min_width": null,
-            "object_fit": null,
-            "object_position": null,
-            "order": null,
-            "overflow": null,
-            "overflow_x": null,
-            "overflow_y": null,
-            "padding": null,
-            "right": null,
-            "top": null,
-            "visibility": null,
-            "width": null
-          }
-        },
-        "277101c35a784e6caf455a13cd9b8e59": {
-          "model_module": "@jupyter-widgets/base",
-          "model_module_version": "1.2.0",
-          "model_name": "LayoutModel",
-          "state": {
-            "_model_module": "@jupyter-widgets/base",
-            "_model_module_version": "1.2.0",
-            "_model_name": "LayoutModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/base",
-            "_view_module_version": "1.2.0",
-            "_view_name": "LayoutView",
-            "align_content": null,
-            "align_items": null,
-            "align_self": null,
-            "border": null,
-            "bottom": null,
-            "display": null,
-            "flex": null,
-            "flex_flow": null,
-            "grid_area": null,
-            "grid_auto_columns": null,
-            "grid_auto_flow": null,
-            "grid_auto_rows": null,
-            "grid_column": null,
-            "grid_gap": null,
-            "grid_row": null,
-            "grid_template_areas": null,
-            "grid_template_columns": null,
-            "grid_template_rows": null,
-            "height": null,
-            "justify_content": null,
-            "justify_items": null,
-            "left": null,
-            "margin": null,
-            "max_height": null,
-            "max_width": null,
-            "min_height": null,
-            "min_width": null,
-            "object_fit": null,
-            "object_position": null,
-            "order": null,
-            "overflow": null,
-            "overflow_x": null,
-            "overflow_y": null,
-            "padding": null,
-            "right": null,
-            "top": null,
-            "visibility": null,
-            "width": null
-          }
-        },
-        "2924814bab5748ddbeeedc70d324195e": {
-          "model_module": "@jupyter-widgets/controls",
-          "model_module_version": "1.5.0",
-          "model_name": "HBoxModel",
-          "state": {
-            "_dom_classes": [],
-            "_model_module": "@jupyter-widgets/controls",
-            "_model_module_version": "1.5.0",
-            "_model_name": "HBoxModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/controls",
-            "_view_module_version": "1.5.0",
-            "_view_name": "HBoxView",
-            "box_style": "",
-            "children": [
-              "IPY_MODEL_4738bccc6b384da5a20a8bcd61ecec59",
-              "IPY_MODEL_044d6d8dda1c4935b1752a9c71c6ee4a",
-              "IPY_MODEL_9277709ad9154d7b8f37d08db84ee425"
-            ],
-            "layout": "IPY_MODEL_f3f1f2487d6f455caeb6ec71a2d51ee2"
-          }
-        },
-        "2958af7c9cdb46038e0336d6b7c6773e": {
-          "model_module": "@jupyter-widgets/controls",
-          "model_module_version": "1.5.0",
-          "model_name": "DescriptionStyleModel",
-          "state": {
-            "_model_module": "@jupyter-widgets/controls",
-            "_model_module_version": "1.5.0",
-            "_model_name": "DescriptionStyleModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/base",
-            "_view_module_version": "1.2.0",
-            "_view_name": "StyleView",
-            "description_width": ""
-          }
-        },
-        "351928faa62543128e0bd29bf89bbf79": {
-          "model_module": "@jupyter-widgets/controls",
-          "model_module_version": "1.5.0",
-          "model_name": "DescriptionStyleModel",
-          "state": {
-            "_model_module": "@jupyter-widgets/controls",
-            "_model_module_version": "1.5.0",
-            "_model_name": "DescriptionStyleModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/base",
-            "_view_module_version": "1.2.0",
-            "_view_name": "StyleView",
-            "description_width": ""
-          }
-        },
-        "38897429b7cf4077aea3a981593ca866": {
-          "model_module": "@jupyter-widgets/controls",
-          "model_module_version": "1.5.0",
-          "model_name": "DescriptionStyleModel",
-          "state": {
-            "_model_module": "@jupyter-widgets/controls",
-            "_model_module_version": "1.5.0",
-            "_model_name": "DescriptionStyleModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/base",
-            "_view_module_version": "1.2.0",
-            "_view_name": "StyleView",
-            "description_width": ""
-          }
-        },
-        "3978f618c4f8467eb83c63a8f5aef98a": {
-          "model_module": "@jupyter-widgets/controls",
-          "model_module_version": "1.5.0",
-          "model_name": "ProgressStyleModel",
-          "state": {
-            "_model_module": "@jupyter-widgets/controls",
-            "_model_module_version": "1.5.0",
-            "_model_name": "ProgressStyleModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/base",
-            "_view_module_version": "1.2.0",
-            "_view_name": "StyleView",
-            "bar_color": null,
-            "description_width": ""
-          }
-        },
-        "3d0376d2e574410eb4ef963d51cac0a6": {
+        "04804c74e1dd43449d5f758cf5d0ba5e": {
           "model_module": "@jupyter-widgets/controls",
           "model_module_version": "1.5.0",
           "model_name": "FloatProgressModel",
@@ -3054,15 +4006,453 @@
             "bar_style": "success",
             "description": "",
             "description_tooltip": null,
-            "layout": "IPY_MODEL_9054d3825edb49cb9c35d24023f50c03",
-            "max": 1,
+            "layout": "IPY_MODEL_f023175de68445f98a6b01bb40ccdc6d",
+            "max": 112,
             "min": 0,
             "orientation": "horizontal",
-            "style": "IPY_MODEL_3978f618c4f8467eb83c63a8f5aef98a",
-            "value": 1
+            "style": "IPY_MODEL_7389b79a0ff44cd68c7866995d728023",
+            "value": 112
           }
         },
-        "425c6c0eaed741669551b9af77096c6f": {
+        "07ce54c75e76488ba4019a20b3707061": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "08f9d125018b41c582a0fa1e234315f9": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_5472af91737446f4a4a2d92a3f684a45",
+            "placeholder": "​",
+            "style": "IPY_MODEL_9fb4368802da4a5a8101ba200d98403a",
+            "value": " 232k/232k [00:00<00:00, 3.18MB/s]"
+          }
+        },
+        "0ac8e976a32c4f5989392b8088546e00": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "0b276315be4345be83da1e03905c8495": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "0c2e30d78c234b1b8098d879442d3bac": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "0c359bc4c94c46acbc9094354a15c33d": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "0e1b9910a77d4b7fa69cb8926e6547d7": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "0e695245b97c4bbc85e349fda3dc07b9": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_90432ec1c24b4607a935c94e130cd68d",
+            "placeholder": "​",
+            "style": "IPY_MODEL_464147b149824f20afc727751a702fc7",
+            "value": "README.md: 100%"
+          }
+        },
+        "0f8bab6b8ed04774b386fe952aae66f1": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "101288236cff40b8bb9dbad80dbbc7ee": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "FloatProgressModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "FloatProgressModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "ProgressView",
+            "bar_style": "success",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_0f8bab6b8ed04774b386fe952aae66f1",
+            "max": 116,
+            "min": 0,
+            "orientation": "horizontal",
+            "style": "IPY_MODEL_cfcb6e456c354d99be91f161552f3376",
+            "value": 116
+          }
+        },
+        "10bc8be68b5545fd8609824b02499ebf": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "1231b9e4cab34c33a38bee63543f1e75": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "13eee164dc534424acb9dc9ee37a9465": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "15ae23892b634a9f821a8fcee14e500b": {
           "model_module": "@jupyter-widgets/controls",
           "model_module_version": "1.5.0",
           "model_name": "HBoxModel",
@@ -3077,14 +4467,406 @@
             "_view_name": "HBoxView",
             "box_style": "",
             "children": [
-              "IPY_MODEL_d124b09896934d289df649375f455a8e",
-              "IPY_MODEL_554cff1a83d44bd2bbd36fd43acac7e2",
-              "IPY_MODEL_d0381718fc8b49a6ac7e7fe85cabba90"
+              "IPY_MODEL_b28d46c2ecdd46b9b3f2da871afbf1cb",
+              "IPY_MODEL_4b83e3caa8ec47169dca04ee9599adeb",
+              "IPY_MODEL_c83c23161674484e81f0db9856c23eb6"
             ],
-            "layout": "IPY_MODEL_fd3daaf9093d45d8a9d39b87835f4582"
+            "layout": "IPY_MODEL_3ded85d9c34246e88f8ce693eb8025e5"
           }
         },
-        "457374ae3035496eb943ad21484f76a0": {
+        "1817f6732a5f44c7adc75a644b1acef2": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "ProgressStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "ProgressStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "bar_color": null,
+            "description_width": ""
+          }
+        },
+        "1a277abd5ea44253bc6894bef258b52b": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_670905a55b19458da69f83c8bcd511d1",
+            "placeholder": "​",
+            "style": "IPY_MODEL_ff54451a48394faaaa9d8cdb690d0718",
+            "value": "tokenizer.json: 100%"
+          }
+        },
+        "1e56da93bcf64ff490416d2b66cd3dc0": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "1e6009b9b0684b8fbaa379ea96f111ee": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "ProgressStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "ProgressStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "bar_color": null,
+            "description_width": ""
+          }
+        },
+        "1e836106837c4ac7a11b36e700c46b64": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_9e4d0fbb51284a7487c495c7b95a293d",
+            "placeholder": "​",
+            "style": "IPY_MODEL_b0f8cf1f79e04b5fb47a810f2c81bd7e",
+            "value": "config.json: 100%"
+          }
+        },
+        "20a66f9de4ed41c7ac9a8e817898ed9e": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_2b2046db907349798e3ae774c15b25d2",
+            "placeholder": "​",
+            "style": "IPY_MODEL_3c18f449359f422f950543bd976fe323",
+            "value": " 1/1 [00:00<00:00, 18.91it/s]"
+          }
+        },
+        "2256ddab0ae1408abb10ba211a08f794": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "ProgressStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "ProgressStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "bar_color": null,
+            "description_width": ""
+          }
+        },
+        "22a665deff88477b9372c0350c4c572b": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "23b0b2f4f82c4a21846e91d7cea91da5": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "254ce460ce244c99a5afe39d5d51f6b7": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "2574b07e4af24715aa89d048cc84e358": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_7551b282ef3a4387a801637de2d5c76e",
+            "placeholder": "​",
+            "style": "IPY_MODEL_69e5263c812c4542a9e5c31fefaa37fe",
+            "value": " 1/1 [00:00<00:00, 15.08it/s]"
+          }
+        },
+        "269b1ad9dc7b4ebb94d7364c75f3f324": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "26f1430ca7cb4ad5b1b8df1ffdbd32a9": {
           "model_module": "@jupyter-widgets/controls",
           "model_module_version": "1.5.0",
           "model_name": "HBoxModel",
@@ -3099,14 +4881,14 @@
             "_view_name": "HBoxView",
             "box_style": "",
             "children": [
-              "IPY_MODEL_bcf4679dda2d4767a0a24cbf236ca76e",
-              "IPY_MODEL_6e4ce98853c84beca11471e7ea9d97df",
-              "IPY_MODEL_186682be50c148c0826fa7c314087562"
+              "IPY_MODEL_7cd2d9c9ea7b4d70902ffaff33033078",
+              "IPY_MODEL_101288236cff40b8bb9dbad80dbbc7ee",
+              "IPY_MODEL_d5c9977838a249eeab6ef628279b8155"
             ],
-            "layout": "IPY_MODEL_e1ef246e3e6c4359b7b61c341119e121"
+            "layout": "IPY_MODEL_d032d1e7b4b54ba28ac83c1a12b23876"
           }
         },
-        "45b569d733f944d29cefae8a5d13b215": {
+        "288c9da81b3c4d80a4959753da973f58": {
           "model_module": "@jupyter-widgets/base",
           "model_module_version": "1.2.0",
           "model_name": "LayoutModel",
@@ -3158,7 +4940,7 @@
             "width": null
           }
         },
-        "4738bccc6b384da5a20a8bcd61ecec59": {
+        "29212208db6b432eb4f708cd64258954": {
           "model_module": "@jupyter-widgets/controls",
           "model_module_version": "1.5.0",
           "model_name": "HTMLModel",
@@ -3173,13 +4955,34 @@
             "_view_name": "HTMLView",
             "description": "",
             "description_tooltip": null,
-            "layout": "IPY_MODEL_66c92a8a89234a61a8c688cf1c3e29a1",
+            "layout": "IPY_MODEL_ef4f63fe9d8f4683a9d20becb6e4e2cb",
             "placeholder": "​",
-            "style": "IPY_MODEL_ee1f4a0c85e44a3b849283337743a8d4",
-            "value": "Batches: 100%"
+            "style": "IPY_MODEL_7508f10c13634e7aa682cfb29c48d9e7",
+            "value": " 349/349 [00:00<00:00, 19.2kB/s]"
           }
         },
-        "4a405d391b974e58a2c4fe00d4bb5815": {
+        "29683ef34d5646c687118a2a0cdec6d4": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_8d370762fafd4d7887ff68ea8279d083",
+            "placeholder": "​",
+            "style": "IPY_MODEL_b6a0eb553b024a71b737ff47ca8f7633",
+            "value": " 1/1 [00:01<00:00,  1.24s/it]"
+          }
+        },
+        "2b2046db907349798e3ae774c15b25d2": {
           "model_module": "@jupyter-widgets/base",
           "model_module_version": "1.2.0",
           "model_name": "LayoutModel",
@@ -3231,61 +5034,7 @@
             "width": null
           }
         },
-        "4ad57f5d8a824afab639e8606ee43ca6": {
-          "model_module": "@jupyter-widgets/controls",
-          "model_module_version": "1.5.0",
-          "model_name": "DescriptionStyleModel",
-          "state": {
-            "_model_module": "@jupyter-widgets/controls",
-            "_model_module_version": "1.5.0",
-            "_model_name": "DescriptionStyleModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/base",
-            "_view_module_version": "1.2.0",
-            "_view_name": "StyleView",
-            "description_width": ""
-          }
-        },
-        "53865d3f918e468ab53504133b127973": {
-          "model_module": "@jupyter-widgets/controls",
-          "model_module_version": "1.5.0",
-          "model_name": "DescriptionStyleModel",
-          "state": {
-            "_model_module": "@jupyter-widgets/controls",
-            "_model_module_version": "1.5.0",
-            "_model_name": "DescriptionStyleModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/base",
-            "_view_module_version": "1.2.0",
-            "_view_name": "StyleView",
-            "description_width": ""
-          }
-        },
-        "554cff1a83d44bd2bbd36fd43acac7e2": {
-          "model_module": "@jupyter-widgets/controls",
-          "model_module_version": "1.5.0",
-          "model_name": "FloatProgressModel",
-          "state": {
-            "_dom_classes": [],
-            "_model_module": "@jupyter-widgets/controls",
-            "_model_module_version": "1.5.0",
-            "_model_name": "FloatProgressModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/controls",
-            "_view_module_version": "1.5.0",
-            "_view_name": "ProgressView",
-            "bar_style": "success",
-            "description": "",
-            "description_tooltip": null,
-            "layout": "IPY_MODEL_6c60c8291e734f549e6c5a46b427b974",
-            "max": 1,
-            "min": 0,
-            "orientation": "horizontal",
-            "style": "IPY_MODEL_de88640505c24928904a3c76bda31c70",
-            "value": 1
-          }
-        },
-        "5afdb88e0159462e98773560e3dad439": {
+        "2e713bcc372e48b2a006558db4d1df68": {
           "model_module": "@jupyter-widgets/controls",
           "model_module_version": "1.5.0",
           "model_name": "HBoxModel",
@@ -3300,14 +5049,333 @@
             "_view_name": "HBoxView",
             "box_style": "",
             "children": [
-              "IPY_MODEL_f7bc4df675a141e380d965138552a142",
-              "IPY_MODEL_d7bf8b49145843ac98a6de424e628729",
-              "IPY_MODEL_8fb17faf68524de2b73321d71b80b407"
+              "IPY_MODEL_1a277abd5ea44253bc6894bef258b52b",
+              "IPY_MODEL_b3eedd82e7da4ce8b3ded70e49a2afd0",
+              "IPY_MODEL_6f5c18cb8002471f8b3764effee37324"
             ],
-            "layout": "IPY_MODEL_45b569d733f944d29cefae8a5d13b215"
+            "layout": "IPY_MODEL_3bebac362b344e8d9103c5011613f1ea"
           }
         },
-        "5cb841b49eaa429e8616ec4b78f501e9": {
+        "2eff72cbd9bb4f1ca77213602caa9417": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HBoxModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HBoxModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HBoxView",
+            "box_style": "",
+            "children": [
+              "IPY_MODEL_e82b5196209f4b9f919c7abb402a4504",
+              "IPY_MODEL_fe34706489c14253a5015ff6332ec4e0",
+              "IPY_MODEL_2574b07e4af24715aa89d048cc84e358"
+            ],
+            "layout": "IPY_MODEL_10bc8be68b5545fd8609824b02499ebf"
+          }
+        },
+        "30798f87a8b848d783fdacd71af5dc04": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "321fce57c158432abeae496ae8a947aa": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "327ff8f5292d47afbfebd3beea187739": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "36b5bc19b2d0407f8ab28ff0da2ce12d": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "3703041a499c426bb427ee008c81cde5": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HBoxModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HBoxModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HBoxView",
+            "box_style": "",
+            "children": [
+              "IPY_MODEL_4b22bbacb995425fb32a2368f3685a92",
+              "IPY_MODEL_49a66eeb9ef74de5ab8904fd90eb7558",
+              "IPY_MODEL_08f9d125018b41c582a0fa1e234315f9"
+            ],
+            "layout": "IPY_MODEL_736c770230644894b85dbc34bd8f1d52"
+          }
+        },
+        "3bebac362b344e8d9103c5011613f1ea": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "3c18f449359f422f950543bd976fe323": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "3cb06377e4454f009d6b2aa7aa6ff0a9": {
           "model_module": "@jupyter-widgets/controls",
           "model_module_version": "1.5.0",
           "model_name": "ProgressStyleModel",
@@ -3323,23 +5391,193 @@
             "description_width": ""
           }
         },
-        "5f19dab8c6da4050bc47fd78838f7530": {
+        "3ded85d9c34246e88f8ce693eb8025e5": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "3ebe00201bdb4e119e3b74f684a58345": {
           "model_module": "@jupyter-widgets/controls",
           "model_module_version": "1.5.0",
-          "model_name": "ProgressStyleModel",
+          "model_name": "DescriptionStyleModel",
           "state": {
             "_model_module": "@jupyter-widgets/controls",
             "_model_module_version": "1.5.0",
-            "_model_name": "ProgressStyleModel",
+            "_model_name": "DescriptionStyleModel",
             "_view_count": null,
             "_view_module": "@jupyter-widgets/base",
             "_view_module_version": "1.2.0",
             "_view_name": "StyleView",
-            "bar_color": null,
             "description_width": ""
           }
         },
-        "6259ffc3ef674df985fd3fa4334f9c8e": {
+        "3ec694106303491ea112a257309bc69c": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "42335bcbc6ee40a79d36c5159cc7da06": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "4282ee7d947e426ba863df9970e82f3f": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "44e34588d6854737b0fb14b4b6a62a95": {
           "model_module": "@jupyter-widgets/controls",
           "model_module_version": "1.5.0",
           "model_name": "HTMLModel",
@@ -3354,13 +5592,13 @@
             "_view_name": "HTMLView",
             "description": "",
             "description_tooltip": null,
-            "layout": "IPY_MODEL_4a405d391b974e58a2c4fe00d4bb5815",
+            "layout": "IPY_MODEL_631c9a95127244c79875c829a7637df6",
             "placeholder": "​",
-            "style": "IPY_MODEL_2958af7c9cdb46038e0336d6b7c6773e",
+            "style": "IPY_MODEL_d25492ad867141bfa8d957d2464b8639",
             "value": "Batches: 100%"
           }
         },
-        "63f34c3d43bb4fdd9faeb6161fd77285": {
+        "4502477db4d948e693012364c2dcb370": {
           "model_module": "@jupyter-widgets/base",
           "model_module_version": "1.2.0",
           "model_name": "LayoutModel",
@@ -3412,111 +5650,75 @@
             "width": null
           }
         },
-        "66c92a8a89234a61a8c688cf1c3e29a1": {
-          "model_module": "@jupyter-widgets/base",
-          "model_module_version": "1.2.0",
-          "model_name": "LayoutModel",
+        "464147b149824f20afc727751a702fc7": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
           "state": {
-            "_model_module": "@jupyter-widgets/base",
-            "_model_module_version": "1.2.0",
-            "_model_name": "LayoutModel",
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
             "_view_count": null,
             "_view_module": "@jupyter-widgets/base",
             "_view_module_version": "1.2.0",
-            "_view_name": "LayoutView",
-            "align_content": null,
-            "align_items": null,
-            "align_self": null,
-            "border": null,
-            "bottom": null,
-            "display": null,
-            "flex": null,
-            "flex_flow": null,
-            "grid_area": null,
-            "grid_auto_columns": null,
-            "grid_auto_flow": null,
-            "grid_auto_rows": null,
-            "grid_column": null,
-            "grid_gap": null,
-            "grid_row": null,
-            "grid_template_areas": null,
-            "grid_template_columns": null,
-            "grid_template_rows": null,
-            "height": null,
-            "justify_content": null,
-            "justify_items": null,
-            "left": null,
-            "margin": null,
-            "max_height": null,
-            "max_width": null,
-            "min_height": null,
-            "min_width": null,
-            "object_fit": null,
-            "object_position": null,
-            "order": null,
-            "overflow": null,
-            "overflow_x": null,
-            "overflow_y": null,
-            "padding": null,
-            "right": null,
-            "top": null,
-            "visibility": null,
-            "width": null
+            "_view_name": "StyleView",
+            "description_width": ""
           }
         },
-        "6c60c8291e734f549e6c5a46b427b974": {
-          "model_module": "@jupyter-widgets/base",
-          "model_module_version": "1.2.0",
-          "model_name": "LayoutModel",
+        "4709067f3f554b93b3ef35e3f58cbf85": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "ProgressStyleModel",
           "state": {
-            "_model_module": "@jupyter-widgets/base",
-            "_model_module_version": "1.2.0",
-            "_model_name": "LayoutModel",
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "ProgressStyleModel",
             "_view_count": null,
             "_view_module": "@jupyter-widgets/base",
             "_view_module_version": "1.2.0",
-            "_view_name": "LayoutView",
-            "align_content": null,
-            "align_items": null,
-            "align_self": null,
-            "border": null,
-            "bottom": null,
-            "display": null,
-            "flex": null,
-            "flex_flow": null,
-            "grid_area": null,
-            "grid_auto_columns": null,
-            "grid_auto_flow": null,
-            "grid_auto_rows": null,
-            "grid_column": null,
-            "grid_gap": null,
-            "grid_row": null,
-            "grid_template_areas": null,
-            "grid_template_columns": null,
-            "grid_template_rows": null,
-            "height": null,
-            "justify_content": null,
-            "justify_items": null,
-            "left": null,
-            "margin": null,
-            "max_height": null,
-            "max_width": null,
-            "min_height": null,
-            "min_width": null,
-            "object_fit": null,
-            "object_position": null,
-            "order": null,
-            "overflow": null,
-            "overflow_x": null,
-            "overflow_y": null,
-            "padding": null,
-            "right": null,
-            "top": null,
-            "visibility": null,
-            "width": null
+            "_view_name": "StyleView",
+            "bar_color": null,
+            "description_width": ""
           }
         },
-        "6e4ce98853c84beca11471e7ea9d97df": {
+        "472b1acc4c5a4c48b2ec62be42d1830c": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HBoxModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HBoxModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HBoxView",
+            "box_style": "",
+            "children": [
+              "IPY_MODEL_44e34588d6854737b0fb14b4b6a62a95",
+              "IPY_MODEL_03402ad03418435ca7a550e3246cd300",
+              "IPY_MODEL_811f115733b14ab4b242a8b11526016c"
+            ],
+            "layout": "IPY_MODEL_e61fdef1dc4b4d809168c0b441b0e6ac"
+          }
+        },
+        "47cf4b6b835d43388576a2abf4cc54f8": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "49a66eeb9ef74de5ab8904fd90eb7558": {
           "model_module": "@jupyter-widgets/controls",
           "model_module_version": "1.5.0",
           "model_name": "FloatProgressModel",
@@ -3532,15 +5734,36 @@
             "bar_style": "success",
             "description": "",
             "description_tooltip": null,
-            "layout": "IPY_MODEL_a0ac7ee92d994c7b9b74e580ab2acdf7",
-            "max": 1,
+            "layout": "IPY_MODEL_1e56da93bcf64ff490416d2b66cd3dc0",
+            "max": 231508,
             "min": 0,
             "orientation": "horizontal",
-            "style": "IPY_MODEL_118b359b83304ae59fad57e28f621645",
-            "value": 1
+            "style": "IPY_MODEL_b7e35038ce344110b785753b655130f5",
+            "value": 231508
           }
         },
-        "6ede3649e8c24015b3ca77490568bfcd": {
+        "4b22bbacb995425fb32a2368f3685a92": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_b67cbbf32f844a19b219be612d5038c9",
+            "placeholder": "​",
+            "style": "IPY_MODEL_774b513d64524ac7823a2cf13efa8d41",
+            "value": "vocab.txt: 100%"
+          }
+        },
+        "4b83e3caa8ec47169dca04ee9599adeb": {
           "model_module": "@jupyter-widgets/controls",
           "model_module_version": "1.5.0",
           "model_name": "FloatProgressModel",
@@ -3556,15 +5779,15 @@
             "bar_style": "success",
             "description": "",
             "description_tooltip": null,
-            "layout": "IPY_MODEL_f10237315e794539a00ca82bfff930be",
+            "layout": "IPY_MODEL_269b1ad9dc7b4ebb94d7364c75f3f324",
             "max": 1,
             "min": 0,
             "orientation": "horizontal",
-            "style": "IPY_MODEL_ca09d2207b00456da4c37b5a782a190c",
+            "style": "IPY_MODEL_2256ddab0ae1408abb10ba211a08f794",
             "value": 1
           }
         },
-        "753dbe7891a143118b55eccf8c252e03": {
+        "4cf1dc345ace4da59f978f661487f975": {
           "model_module": "@jupyter-widgets/base",
           "model_module_version": "1.2.0",
           "model_name": "LayoutModel",
@@ -3616,28 +5839,7 @@
             "width": null
           }
         },
-        "8fb17faf68524de2b73321d71b80b407": {
-          "model_module": "@jupyter-widgets/controls",
-          "model_module_version": "1.5.0",
-          "model_name": "HTMLModel",
-          "state": {
-            "_dom_classes": [],
-            "_model_module": "@jupyter-widgets/controls",
-            "_model_module_version": "1.5.0",
-            "_model_name": "HTMLModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/controls",
-            "_view_module_version": "1.5.0",
-            "_view_name": "HTMLView",
-            "description": "",
-            "description_tooltip": null,
-            "layout": "IPY_MODEL_277101c35a784e6caf455a13cd9b8e59",
-            "placeholder": "​",
-            "style": "IPY_MODEL_d06666f765764f949e1876f2d5d67242",
-            "value": " 1/1 [00:01<00:00,  1.68s/it]"
-          }
-        },
-        "9054d3825edb49cb9c35d24023f50c03": {
+        "50dd8994a4cf486ebbec5ffd4322992a": {
           "model_module": "@jupyter-widgets/base",
           "model_module_version": "1.2.0",
           "model_name": "LayoutModel",
@@ -3689,205 +5891,7 @@
             "width": null
           }
         },
-        "92135b9cb201475681ee0886887c84a8": {
-          "model_module": "@jupyter-widgets/base",
-          "model_module_version": "1.2.0",
-          "model_name": "LayoutModel",
-          "state": {
-            "_model_module": "@jupyter-widgets/base",
-            "_model_module_version": "1.2.0",
-            "_model_name": "LayoutModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/base",
-            "_view_module_version": "1.2.0",
-            "_view_name": "LayoutView",
-            "align_content": null,
-            "align_items": null,
-            "align_self": null,
-            "border": null,
-            "bottom": null,
-            "display": null,
-            "flex": null,
-            "flex_flow": null,
-            "grid_area": null,
-            "grid_auto_columns": null,
-            "grid_auto_flow": null,
-            "grid_auto_rows": null,
-            "grid_column": null,
-            "grid_gap": null,
-            "grid_row": null,
-            "grid_template_areas": null,
-            "grid_template_columns": null,
-            "grid_template_rows": null,
-            "height": null,
-            "justify_content": null,
-            "justify_items": null,
-            "left": null,
-            "margin": null,
-            "max_height": null,
-            "max_width": null,
-            "min_height": null,
-            "min_width": null,
-            "object_fit": null,
-            "object_position": null,
-            "order": null,
-            "overflow": null,
-            "overflow_x": null,
-            "overflow_y": null,
-            "padding": null,
-            "right": null,
-            "top": null,
-            "visibility": null,
-            "width": null
-          }
-        },
-        "9277709ad9154d7b8f37d08db84ee425": {
-          "model_module": "@jupyter-widgets/controls",
-          "model_module_version": "1.5.0",
-          "model_name": "HTMLModel",
-          "state": {
-            "_dom_classes": [],
-            "_model_module": "@jupyter-widgets/controls",
-            "_model_module_version": "1.5.0",
-            "_model_name": "HTMLModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/controls",
-            "_view_module_version": "1.5.0",
-            "_view_name": "HTMLView",
-            "description": "",
-            "description_tooltip": null,
-            "layout": "IPY_MODEL_a447ea9af3e14e5e94eb14ed8dd3c0de",
-            "placeholder": "​",
-            "style": "IPY_MODEL_0243626d7ef44ef2b90e8fed5c13183d",
-            "value": " 1/1 [00:02<00:00,  2.65s/it]"
-          }
-        },
-        "a0ac7ee92d994c7b9b74e580ab2acdf7": {
-          "model_module": "@jupyter-widgets/base",
-          "model_module_version": "1.2.0",
-          "model_name": "LayoutModel",
-          "state": {
-            "_model_module": "@jupyter-widgets/base",
-            "_model_module_version": "1.2.0",
-            "_model_name": "LayoutModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/base",
-            "_view_module_version": "1.2.0",
-            "_view_name": "LayoutView",
-            "align_content": null,
-            "align_items": null,
-            "align_self": null,
-            "border": null,
-            "bottom": null,
-            "display": null,
-            "flex": null,
-            "flex_flow": null,
-            "grid_area": null,
-            "grid_auto_columns": null,
-            "grid_auto_flow": null,
-            "grid_auto_rows": null,
-            "grid_column": null,
-            "grid_gap": null,
-            "grid_row": null,
-            "grid_template_areas": null,
-            "grid_template_columns": null,
-            "grid_template_rows": null,
-            "height": null,
-            "justify_content": null,
-            "justify_items": null,
-            "left": null,
-            "margin": null,
-            "max_height": null,
-            "max_width": null,
-            "min_height": null,
-            "min_width": null,
-            "object_fit": null,
-            "object_position": null,
-            "order": null,
-            "overflow": null,
-            "overflow_x": null,
-            "overflow_y": null,
-            "padding": null,
-            "right": null,
-            "top": null,
-            "visibility": null,
-            "width": null
-          }
-        },
-        "a0be415018644c3cac098ab9b19c2391": {
-          "model_module": "@jupyter-widgets/controls",
-          "model_module_version": "1.5.0",
-          "model_name": "HTMLModel",
-          "state": {
-            "_dom_classes": [],
-            "_model_module": "@jupyter-widgets/controls",
-            "_model_module_version": "1.5.0",
-            "_model_name": "HTMLModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/controls",
-            "_view_module_version": "1.5.0",
-            "_view_name": "HTMLView",
-            "description": "",
-            "description_tooltip": null,
-            "layout": "IPY_MODEL_e4b1dfe159304c5f88766b33e85a5c19",
-            "placeholder": "​",
-            "style": "IPY_MODEL_2100363a158b4488a58620983aa5bdd4",
-            "value": "Batches: 100%"
-          }
-        },
-        "a447ea9af3e14e5e94eb14ed8dd3c0de": {
-          "model_module": "@jupyter-widgets/base",
-          "model_module_version": "1.2.0",
-          "model_name": "LayoutModel",
-          "state": {
-            "_model_module": "@jupyter-widgets/base",
-            "_model_module_version": "1.2.0",
-            "_model_name": "LayoutModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/base",
-            "_view_module_version": "1.2.0",
-            "_view_name": "LayoutView",
-            "align_content": null,
-            "align_items": null,
-            "align_self": null,
-            "border": null,
-            "bottom": null,
-            "display": null,
-            "flex": null,
-            "flex_flow": null,
-            "grid_area": null,
-            "grid_auto_columns": null,
-            "grid_auto_flow": null,
-            "grid_auto_rows": null,
-            "grid_column": null,
-            "grid_gap": null,
-            "grid_row": null,
-            "grid_template_areas": null,
-            "grid_template_columns": null,
-            "grid_template_rows": null,
-            "height": null,
-            "justify_content": null,
-            "justify_items": null,
-            "left": null,
-            "margin": null,
-            "max_height": null,
-            "max_width": null,
-            "min_height": null,
-            "min_width": null,
-            "object_fit": null,
-            "object_position": null,
-            "order": null,
-            "overflow": null,
-            "overflow_x": null,
-            "overflow_y": null,
-            "padding": null,
-            "right": null,
-            "top": null,
-            "visibility": null,
-            "width": null
-          }
-        },
-        "a6a1eb412f204578b80e5b6717c1e3a5": {
+        "52fe404ec9c14db2a7279b4c154eef3d": {
           "model_module": "@jupyter-widgets/controls",
           "model_module_version": "1.5.0",
           "model_name": "DescriptionStyleModel",
@@ -3902,7 +5906,7 @@
             "description_width": ""
           }
         },
-        "ab1f339cba094c918fc5507f8361de5c": {
+        "541b9b4e74614e2cb855bb90f03df538": {
           "model_module": "@jupyter-widgets/base",
           "model_module_version": "1.2.0",
           "model_name": "LayoutModel",
@@ -3954,189 +5958,7 @@
             "width": null
           }
         },
-        "b66984cc5de541a5801a1e6e54d40daf": {
-          "model_module": "@jupyter-widgets/controls",
-          "model_module_version": "1.5.0",
-          "model_name": "HTMLModel",
-          "state": {
-            "_dom_classes": [],
-            "_model_module": "@jupyter-widgets/controls",
-            "_model_module_version": "1.5.0",
-            "_model_name": "HTMLModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/controls",
-            "_view_module_version": "1.5.0",
-            "_view_name": "HTMLView",
-            "description": "",
-            "description_tooltip": null,
-            "layout": "IPY_MODEL_efd68f6dc0b3428e8f5fc830c1bf2341",
-            "placeholder": "​",
-            "style": "IPY_MODEL_4ad57f5d8a824afab639e8606ee43ca6",
-            "value": " 1/1 [00:00<00:00,  5.36it/s]"
-          }
-        },
-        "bbb93c771a9c453bb90e729b1f73b931": {
-          "model_module": "@jupyter-widgets/base",
-          "model_module_version": "1.2.0",
-          "model_name": "LayoutModel",
-          "state": {
-            "_model_module": "@jupyter-widgets/base",
-            "_model_module_version": "1.2.0",
-            "_model_name": "LayoutModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/base",
-            "_view_module_version": "1.2.0",
-            "_view_name": "LayoutView",
-            "align_content": null,
-            "align_items": null,
-            "align_self": null,
-            "border": null,
-            "bottom": null,
-            "display": null,
-            "flex": null,
-            "flex_flow": null,
-            "grid_area": null,
-            "grid_auto_columns": null,
-            "grid_auto_flow": null,
-            "grid_auto_rows": null,
-            "grid_column": null,
-            "grid_gap": null,
-            "grid_row": null,
-            "grid_template_areas": null,
-            "grid_template_columns": null,
-            "grid_template_rows": null,
-            "height": null,
-            "justify_content": null,
-            "justify_items": null,
-            "left": null,
-            "margin": null,
-            "max_height": null,
-            "max_width": null,
-            "min_height": null,
-            "min_width": null,
-            "object_fit": null,
-            "object_position": null,
-            "order": null,
-            "overflow": null,
-            "overflow_x": null,
-            "overflow_y": null,
-            "padding": null,
-            "right": null,
-            "top": null,
-            "visibility": null,
-            "width": null
-          }
-        },
-        "bcf4679dda2d4767a0a24cbf236ca76e": {
-          "model_module": "@jupyter-widgets/controls",
-          "model_module_version": "1.5.0",
-          "model_name": "HTMLModel",
-          "state": {
-            "_dom_classes": [],
-            "_model_module": "@jupyter-widgets/controls",
-            "_model_module_version": "1.5.0",
-            "_model_name": "HTMLModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/controls",
-            "_view_module_version": "1.5.0",
-            "_view_name": "HTMLView",
-            "description": "",
-            "description_tooltip": null,
-            "layout": "IPY_MODEL_bbb93c771a9c453bb90e729b1f73b931",
-            "placeholder": "​",
-            "style": "IPY_MODEL_351928faa62543128e0bd29bf89bbf79",
-            "value": "Batches: 100%"
-          }
-        },
-        "ca09d2207b00456da4c37b5a782a190c": {
-          "model_module": "@jupyter-widgets/controls",
-          "model_module_version": "1.5.0",
-          "model_name": "ProgressStyleModel",
-          "state": {
-            "_model_module": "@jupyter-widgets/controls",
-            "_model_module_version": "1.5.0",
-            "_model_name": "ProgressStyleModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/base",
-            "_view_module_version": "1.2.0",
-            "_view_name": "StyleView",
-            "bar_color": null,
-            "description_width": ""
-          }
-        },
-        "ce7de1af99434ad38a9382e7253dbfc0": {
-          "model_module": "@jupyter-widgets/controls",
-          "model_module_version": "1.5.0",
-          "model_name": "DescriptionStyleModel",
-          "state": {
-            "_model_module": "@jupyter-widgets/controls",
-            "_model_module_version": "1.5.0",
-            "_model_name": "DescriptionStyleModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/base",
-            "_view_module_version": "1.2.0",
-            "_view_name": "StyleView",
-            "description_width": ""
-          }
-        },
-        "d0381718fc8b49a6ac7e7fe85cabba90": {
-          "model_module": "@jupyter-widgets/controls",
-          "model_module_version": "1.5.0",
-          "model_name": "HTMLModel",
-          "state": {
-            "_dom_classes": [],
-            "_model_module": "@jupyter-widgets/controls",
-            "_model_module_version": "1.5.0",
-            "_model_name": "HTMLModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/controls",
-            "_view_module_version": "1.5.0",
-            "_view_name": "HTMLView",
-            "description": "",
-            "description_tooltip": null,
-            "layout": "IPY_MODEL_fc086d0dd1a745308c59ae219ae135c5",
-            "placeholder": "​",
-            "style": "IPY_MODEL_15d3ff07f1c54e58b51d452caca01209",
-            "value": " 1/1 [00:00<00:00, 14.36it/s]"
-          }
-        },
-        "d06666f765764f949e1876f2d5d67242": {
-          "model_module": "@jupyter-widgets/controls",
-          "model_module_version": "1.5.0",
-          "model_name": "DescriptionStyleModel",
-          "state": {
-            "_model_module": "@jupyter-widgets/controls",
-            "_model_module_version": "1.5.0",
-            "_model_name": "DescriptionStyleModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/base",
-            "_view_module_version": "1.2.0",
-            "_view_name": "StyleView",
-            "description_width": ""
-          }
-        },
-        "d124b09896934d289df649375f455a8e": {
-          "model_module": "@jupyter-widgets/controls",
-          "model_module_version": "1.5.0",
-          "model_name": "HTMLModel",
-          "state": {
-            "_dom_classes": [],
-            "_model_module": "@jupyter-widgets/controls",
-            "_model_module_version": "1.5.0",
-            "_model_name": "HTMLModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/controls",
-            "_view_module_version": "1.5.0",
-            "_view_name": "HTMLView",
-            "description": "",
-            "description_tooltip": null,
-            "layout": "IPY_MODEL_753dbe7891a143118b55eccf8c252e03",
-            "placeholder": "​",
-            "style": "IPY_MODEL_ce7de1af99434ad38a9382e7253dbfc0",
-            "value": "Batches: 100%"
-          }
-        },
-        "d7bf8b49145843ac98a6de424e628729": {
+        "5459633eb6e94ec391d13fcf67425726": {
           "model_module": "@jupyter-widgets/controls",
           "model_module_version": "1.5.0",
           "model_name": "FloatProgressModel",
@@ -4152,15 +5974,91 @@
             "bar_style": "success",
             "description": "",
             "description_tooltip": null,
-            "layout": "IPY_MODEL_17603dd7fedf4798a74533fbfd5bb421",
-            "max": 1,
+            "layout": "IPY_MODEL_8e81ae00681347cb906b392c3656a64a",
+            "max": 90868376,
             "min": 0,
             "orientation": "horizontal",
-            "style": "IPY_MODEL_5f19dab8c6da4050bc47fd78838f7530",
-            "value": 1
+            "style": "IPY_MODEL_74bedc38b7da4e8a83b0c892d7aa59b5",
+            "value": 90868376
           }
         },
-        "de88640505c24928904a3c76bda31c70": {
+        "5472af91737446f4a4a2d92a3f684a45": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "55591e8179084fcfa3a61c8bd8d09dcb": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "FloatProgressModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "FloatProgressModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "ProgressView",
+            "bar_style": "success",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_0c359bc4c94c46acbc9094354a15c33d",
+            "max": 612,
+            "min": 0,
+            "orientation": "horizontal",
+            "style": "IPY_MODEL_59d0b59b6c2248508d0601ff13878d33",
+            "value": 612
+          }
+        },
+        "59d0b59b6c2248508d0601ff13878d33": {
           "model_module": "@jupyter-widgets/controls",
           "model_module_version": "1.5.0",
           "model_name": "ProgressStyleModel",
@@ -4176,282 +6074,23 @@
             "description_width": ""
           }
         },
-        "e1ef246e3e6c4359b7b61c341119e121": {
-          "model_module": "@jupyter-widgets/base",
-          "model_module_version": "1.2.0",
-          "model_name": "LayoutModel",
-          "state": {
-            "_model_module": "@jupyter-widgets/base",
-            "_model_module_version": "1.2.0",
-            "_model_name": "LayoutModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/base",
-            "_view_module_version": "1.2.0",
-            "_view_name": "LayoutView",
-            "align_content": null,
-            "align_items": null,
-            "align_self": null,
-            "border": null,
-            "bottom": null,
-            "display": null,
-            "flex": null,
-            "flex_flow": null,
-            "grid_area": null,
-            "grid_auto_columns": null,
-            "grid_auto_flow": null,
-            "grid_auto_rows": null,
-            "grid_column": null,
-            "grid_gap": null,
-            "grid_row": null,
-            "grid_template_areas": null,
-            "grid_template_columns": null,
-            "grid_template_rows": null,
-            "height": null,
-            "justify_content": null,
-            "justify_items": null,
-            "left": null,
-            "margin": null,
-            "max_height": null,
-            "max_width": null,
-            "min_height": null,
-            "min_width": null,
-            "object_fit": null,
-            "object_position": null,
-            "order": null,
-            "overflow": null,
-            "overflow_x": null,
-            "overflow_y": null,
-            "padding": null,
-            "right": null,
-            "top": null,
-            "visibility": null,
-            "width": null
-          }
-        },
-        "e4b1dfe159304c5f88766b33e85a5c19": {
-          "model_module": "@jupyter-widgets/base",
-          "model_module_version": "1.2.0",
-          "model_name": "LayoutModel",
-          "state": {
-            "_model_module": "@jupyter-widgets/base",
-            "_model_module_version": "1.2.0",
-            "_model_name": "LayoutModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/base",
-            "_view_module_version": "1.2.0",
-            "_view_name": "LayoutView",
-            "align_content": null,
-            "align_items": null,
-            "align_self": null,
-            "border": null,
-            "bottom": null,
-            "display": null,
-            "flex": null,
-            "flex_flow": null,
-            "grid_area": null,
-            "grid_auto_columns": null,
-            "grid_auto_flow": null,
-            "grid_auto_rows": null,
-            "grid_column": null,
-            "grid_gap": null,
-            "grid_row": null,
-            "grid_template_areas": null,
-            "grid_template_columns": null,
-            "grid_template_rows": null,
-            "height": null,
-            "justify_content": null,
-            "justify_items": null,
-            "left": null,
-            "margin": null,
-            "max_height": null,
-            "max_width": null,
-            "min_height": null,
-            "min_width": null,
-            "object_fit": null,
-            "object_position": null,
-            "order": null,
-            "overflow": null,
-            "overflow_x": null,
-            "overflow_y": null,
-            "padding": null,
-            "right": null,
-            "top": null,
-            "visibility": null,
-            "width": null
-          }
-        },
-        "ee1f4a0c85e44a3b849283337743a8d4": {
+        "5a620017a5384af1a056de687b2670db": {
           "model_module": "@jupyter-widgets/controls",
           "model_module_version": "1.5.0",
-          "model_name": "DescriptionStyleModel",
+          "model_name": "ProgressStyleModel",
           "state": {
             "_model_module": "@jupyter-widgets/controls",
             "_model_module_version": "1.5.0",
-            "_model_name": "DescriptionStyleModel",
+            "_model_name": "ProgressStyleModel",
             "_view_count": null,
             "_view_module": "@jupyter-widgets/base",
             "_view_module_version": "1.2.0",
             "_view_name": "StyleView",
+            "bar_color": null,
             "description_width": ""
           }
         },
-        "efd68f6dc0b3428e8f5fc830c1bf2341": {
-          "model_module": "@jupyter-widgets/base",
-          "model_module_version": "1.2.0",
-          "model_name": "LayoutModel",
-          "state": {
-            "_model_module": "@jupyter-widgets/base",
-            "_model_module_version": "1.2.0",
-            "_model_name": "LayoutModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/base",
-            "_view_module_version": "1.2.0",
-            "_view_name": "LayoutView",
-            "align_content": null,
-            "align_items": null,
-            "align_self": null,
-            "border": null,
-            "bottom": null,
-            "display": null,
-            "flex": null,
-            "flex_flow": null,
-            "grid_area": null,
-            "grid_auto_columns": null,
-            "grid_auto_flow": null,
-            "grid_auto_rows": null,
-            "grid_column": null,
-            "grid_gap": null,
-            "grid_row": null,
-            "grid_template_areas": null,
-            "grid_template_columns": null,
-            "grid_template_rows": null,
-            "height": null,
-            "justify_content": null,
-            "justify_items": null,
-            "left": null,
-            "margin": null,
-            "max_height": null,
-            "max_width": null,
-            "min_height": null,
-            "min_width": null,
-            "object_fit": null,
-            "object_position": null,
-            "order": null,
-            "overflow": null,
-            "overflow_x": null,
-            "overflow_y": null,
-            "padding": null,
-            "right": null,
-            "top": null,
-            "visibility": null,
-            "width": null
-          }
-        },
-        "f10237315e794539a00ca82bfff930be": {
-          "model_module": "@jupyter-widgets/base",
-          "model_module_version": "1.2.0",
-          "model_name": "LayoutModel",
-          "state": {
-            "_model_module": "@jupyter-widgets/base",
-            "_model_module_version": "1.2.0",
-            "_model_name": "LayoutModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/base",
-            "_view_module_version": "1.2.0",
-            "_view_name": "LayoutView",
-            "align_content": null,
-            "align_items": null,
-            "align_self": null,
-            "border": null,
-            "bottom": null,
-            "display": null,
-            "flex": null,
-            "flex_flow": null,
-            "grid_area": null,
-            "grid_auto_columns": null,
-            "grid_auto_flow": null,
-            "grid_auto_rows": null,
-            "grid_column": null,
-            "grid_gap": null,
-            "grid_row": null,
-            "grid_template_areas": null,
-            "grid_template_columns": null,
-            "grid_template_rows": null,
-            "height": null,
-            "justify_content": null,
-            "justify_items": null,
-            "left": null,
-            "margin": null,
-            "max_height": null,
-            "max_width": null,
-            "min_height": null,
-            "min_width": null,
-            "object_fit": null,
-            "object_position": null,
-            "order": null,
-            "overflow": null,
-            "overflow_x": null,
-            "overflow_y": null,
-            "padding": null,
-            "right": null,
-            "top": null,
-            "visibility": null,
-            "width": null
-          }
-        },
-        "f3f1f2487d6f455caeb6ec71a2d51ee2": {
-          "model_module": "@jupyter-widgets/base",
-          "model_module_version": "1.2.0",
-          "model_name": "LayoutModel",
-          "state": {
-            "_model_module": "@jupyter-widgets/base",
-            "_model_module_version": "1.2.0",
-            "_model_name": "LayoutModel",
-            "_view_count": null,
-            "_view_module": "@jupyter-widgets/base",
-            "_view_module_version": "1.2.0",
-            "_view_name": "LayoutView",
-            "align_content": null,
-            "align_items": null,
-            "align_self": null,
-            "border": null,
-            "bottom": null,
-            "display": null,
-            "flex": null,
-            "flex_flow": null,
-            "grid_area": null,
-            "grid_auto_columns": null,
-            "grid_auto_flow": null,
-            "grid_auto_rows": null,
-            "grid_column": null,
-            "grid_gap": null,
-            "grid_row": null,
-            "grid_template_areas": null,
-            "grid_template_columns": null,
-            "grid_template_rows": null,
-            "height": null,
-            "justify_content": null,
-            "justify_items": null,
-            "left": null,
-            "margin": null,
-            "max_height": null,
-            "max_width": null,
-            "min_height": null,
-            "min_width": null,
-            "object_fit": null,
-            "object_position": null,
-            "order": null,
-            "overflow": null,
-            "overflow_x": null,
-            "overflow_y": null,
-            "padding": null,
-            "right": null,
-            "top": null,
-            "visibility": null,
-            "width": null
-          }
-        },
-        "f7bc4df675a141e380d965138552a142": {
+        "5ce87402a79342af995df41ac3940d55": {
           "model_module": "@jupyter-widgets/controls",
           "model_module_version": "1.5.0",
           "model_name": "HTMLModel",
@@ -4466,13 +6105,1381 @@
             "_view_name": "HTMLView",
             "description": "",
             "description_tooltip": null,
-            "layout": "IPY_MODEL_fdd057a4506f4f119d945bab5b930799",
+            "layout": "IPY_MODEL_f9b768c703494dd198f2978aff4892e8",
             "placeholder": "​",
-            "style": "IPY_MODEL_53865d3f918e468ab53504133b127973",
+            "style": "IPY_MODEL_1231b9e4cab34c33a38bee63543f1e75",
+            "value": "modules.json: 100%"
+          }
+        },
+        "5e535ed2b83e496ab57b1c80b615ab0c": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "5f6014ba13fa4a659b9eb1b5f83599a7": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "61bd0d490c0e4c04a331cf9ce6b7d38f": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "631c9a95127244c79875c829a7637df6": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "670905a55b19458da69f83c8bcd511d1": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "67e37a088be64a2ba786ca923b1017dd": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "69e5263c812c4542a9e5c31fefaa37fe": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "6f5c18cb8002471f8b3764effee37324": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_abce503d70594c2ca9afdc47847c125b",
+            "placeholder": "​",
+            "style": "IPY_MODEL_028e291ee53947bbbbc4bfb68c695f5f",
+            "value": " 466k/466k [00:00<00:00, 3.52MB/s]"
+          }
+        },
+        "722a7fe16af3422585a20c651345cfa4": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HBoxModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HBoxModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HBoxView",
+            "box_style": "",
+            "children": [
+              "IPY_MODEL_f5596c1c9c4d42f3bc171961f9582eff",
+              "IPY_MODEL_85d66e615b5742e78657b1e60c75fc72",
+              "IPY_MODEL_731c02dc5dd446c3b22765575148e256"
+            ],
+            "layout": "IPY_MODEL_254ce460ce244c99a5afe39d5d51f6b7"
+          }
+        },
+        "72e7c092fb054b7ea0dcd2782b5d8a7d": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_327ff8f5292d47afbfebd3beea187739",
+            "placeholder": "​",
+            "style": "IPY_MODEL_988cac4341b646079fc73719f3f88ad7",
+            "value": "tokenizer_config.json: 100%"
+          }
+        },
+        "731c02dc5dd446c3b22765575148e256": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_4502477db4d948e693012364c2dcb370",
+            "placeholder": "​",
+            "style": "IPY_MODEL_52fe404ec9c14db2a7279b4c154eef3d",
+            "value": " 190/190 [00:00<00:00, 12.8kB/s]"
+          }
+        },
+        "736c770230644894b85dbc34bd8f1d52": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "7389b79a0ff44cd68c7866995d728023": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "ProgressStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "ProgressStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "bar_color": null,
+            "description_width": ""
+          }
+        },
+        "74bedc38b7da4e8a83b0c892d7aa59b5": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "ProgressStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "ProgressStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "bar_color": null,
+            "description_width": ""
+          }
+        },
+        "7508f10c13634e7aa682cfb29c48d9e7": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "75307e3dee604d30aa44713e6e293e64": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HBoxModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HBoxModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HBoxView",
+            "box_style": "",
+            "children": [
+              "IPY_MODEL_5ce87402a79342af995df41ac3940d55",
+              "IPY_MODEL_fbbcc19886cc43b38424fbb184162c61",
+              "IPY_MODEL_29212208db6b432eb4f708cd64258954"
+            ],
+            "layout": "IPY_MODEL_50dd8994a4cf486ebbec5ffd4322992a"
+          }
+        },
+        "754deb3970604d48a522bc9f021ad945": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "7551b282ef3a4387a801637de2d5c76e": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "7611cfc7965649ba88ca57c1a9f9ccf3": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "76d37a48a73946bab2821f097cf2605f": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "774b513d64524ac7823a2cf13efa8d41": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "7cc356ed20e94401b72a0e138ad0f5df": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HBoxModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HBoxModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HBoxView",
+            "box_style": "",
+            "children": [
+              "IPY_MODEL_acd39276db17439798a97abc56460b0f",
+              "IPY_MODEL_bda474c3b8184597a6a9bc6da0672a50",
+              "IPY_MODEL_20a66f9de4ed41c7ac9a8e817898ed9e"
+            ],
+            "layout": "IPY_MODEL_e662ba10fbae49d9b66172125dfc0717"
+          }
+        },
+        "7cd2d9c9ea7b4d70902ffaff33033078": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_321fce57c158432abeae496ae8a947aa",
+            "placeholder": "​",
+            "style": "IPY_MODEL_3ebe00201bdb4e119e3b74f684a58345",
+            "value": "config_sentence_transformers.json: 100%"
+          }
+        },
+        "7d8653fca29f4df3a7487733ff9db60b": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "811f115733b14ab4b242a8b11526016c": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_02baf670942347d69c290452de8641e4",
+            "placeholder": "​",
+            "style": "IPY_MODEL_7611cfc7965649ba88ca57c1a9f9ccf3",
+            "value": " 1/1 [00:00<00:00, 13.00it/s]"
+          }
+        },
+        "844b06df5749441fab6f61656ce581a9": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "FloatProgressModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "FloatProgressModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "ProgressView",
+            "bar_style": "success",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_03bbebd659e64b5d9c29a73570c34854",
+            "max": 53,
+            "min": 0,
+            "orientation": "horizontal",
+            "style": "IPY_MODEL_b68e5097d2504d2cbd7e19aa1aac3a04",
+            "value": 53
+          }
+        },
+        "85d66e615b5742e78657b1e60c75fc72": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "FloatProgressModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "FloatProgressModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "ProgressView",
+            "bar_style": "success",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_dd85d37dd1d14c7ea4592f8e11b2d2c8",
+            "max": 190,
+            "min": 0,
+            "orientation": "horizontal",
+            "style": "IPY_MODEL_3cb06377e4454f009d6b2aa7aa6ff0a9",
+            "value": 190
+          }
+        },
+        "861a00796f55470e85d94733eeee9a5f": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_e2e49c25d6fc4592b317e94cfabc2e5e",
+            "placeholder": "​",
+            "style": "IPY_MODEL_76d37a48a73946bab2821f097cf2605f",
+            "value": "model.safetensors: 100%"
+          }
+        },
+        "87700a80125348f28c4f249bdf8b0a8d": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_0e1b9910a77d4b7fa69cb8926e6547d7",
+            "placeholder": "​",
+            "style": "IPY_MODEL_0b276315be4345be83da1e03905c8495",
+            "value": " 10.7k/10.7k [00:00<00:00, 862kB/s]"
+          }
+        },
+        "879e48d9a9e04183903d94ffe98313d2": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "ProgressStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "ProgressStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "bar_color": null,
+            "description_width": ""
+          }
+        },
+        "8902c3622da540e496ed5b1524bd01ca": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "891cb726d45c4fef8f2c74a56df5532b": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "8b1ea80221174fae943d5c9f997dfb57": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "FloatProgressModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "FloatProgressModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "ProgressView",
+            "bar_style": "success",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_900a4dac08f540dfb35c29f63236a12c",
+            "max": 350,
+            "min": 0,
+            "orientation": "horizontal",
+            "style": "IPY_MODEL_1e6009b9b0684b8fbaa379ea96f111ee",
+            "value": 350
+          }
+        },
+        "8d370762fafd4d7887ff68ea8279d083": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "8dee873065a047799a04e49ab791e449": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "FloatProgressModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "FloatProgressModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "ProgressView",
+            "bar_style": "success",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_ec747bd7c37c45298896c513634cd59a",
+            "max": 1,
+            "min": 0,
+            "orientation": "horizontal",
+            "style": "IPY_MODEL_5a620017a5384af1a056de687b2670db",
+            "value": 1
+          }
+        },
+        "8e2b70ffe4eb4974bd6393fcc1292267": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "8e81ae00681347cb906b392c3656a64a": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "8f30fca71bf24e5ca26e17c2321f893c": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "900a4dac08f540dfb35c29f63236a12c": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "90432ec1c24b4607a935c94e130cd68d": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "943f8fcb66614353a51f32f8344b6122": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HBoxModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HBoxModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HBoxView",
+            "box_style": "",
+            "children": [
+              "IPY_MODEL_0e695245b97c4bbc85e349fda3dc07b9",
+              "IPY_MODEL_bb0d168c41f540b8ae42239d3938483a",
+              "IPY_MODEL_87700a80125348f28c4f249bdf8b0a8d"
+            ],
+            "layout": "IPY_MODEL_8902c3622da540e496ed5b1524bd01ca"
+          }
+        },
+        "95a506c3007c4525b01ee4e1600d671b": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_8e2b70ffe4eb4974bd6393fcc1292267",
+            "placeholder": "​",
+            "style": "IPY_MODEL_13eee164dc534424acb9dc9ee37a9465",
+            "value": " 112/112 [00:00<00:00, 8.09kB/s]"
+          }
+        },
+        "980292182c7144e194604c13ac544a26": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_288c9da81b3c4d80a4959753da973f58",
+            "placeholder": "​",
+            "style": "IPY_MODEL_cf453a1ed54645aba656f9a3f1461e69",
             "value": "Batches: 100%"
           }
         },
-        "fc086d0dd1a745308c59ae219ae135c5": {
+        "98786f52ef5345b0b9164b9c1f2b8e18": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "ProgressStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "ProgressStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "bar_color": null,
+            "description_width": ""
+          }
+        },
+        "988cac4341b646079fc73719f3f88ad7": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "9bb8bf12010f42b2b17c10c7ccaa7bf8": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "ProgressStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "ProgressStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "bar_color": null,
+            "description_width": ""
+          }
+        },
+        "9dece059f1204e29b106fca9e191ddb3": {
           "model_module": "@jupyter-widgets/base",
           "model_module_version": "1.2.0",
           "model_name": "LayoutModel",
@@ -4524,7 +7531,7 @@
             "width": null
           }
         },
-        "fd3daaf9093d45d8a9d39b87835f4582": {
+        "9df914248c214597bed7d7980c7a0afe": {
           "model_module": "@jupyter-widgets/base",
           "model_module_version": "1.2.0",
           "model_name": "LayoutModel",
@@ -4576,7 +7583,7 @@
             "width": null
           }
         },
-        "fdd057a4506f4f119d945bab5b930799": {
+        "9e4d0fbb51284a7487c495c7b95a293d": {
           "model_module": "@jupyter-widgets/base",
           "model_module_version": "1.2.0",
           "model_name": "LayoutModel",
@@ -4627,6 +7634,1690 @@
             "visibility": null,
             "width": null
           }
+        },
+        "9fb4368802da4a5a8101ba200d98403a": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "a0d6b0caeb2340fe96c8f5569e3d3ae4": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "a530662719374c95a9bef12e59e28c85": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HBoxModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HBoxModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HBoxView",
+            "box_style": "",
+            "children": [
+              "IPY_MODEL_bffc0f4b12f141398535990709fd4f2c",
+              "IPY_MODEL_04804c74e1dd43449d5f758cf5d0ba5e",
+              "IPY_MODEL_95a506c3007c4525b01ee4e1600d671b"
+            ],
+            "layout": "IPY_MODEL_a0d6b0caeb2340fe96c8f5569e3d3ae4"
+          }
+        },
+        "abce503d70594c2ca9afdc47847c125b": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "abe6cf39b784436993fcbe92221c31a3": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "acd39276db17439798a97abc56460b0f": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_d452b32c54e14e41a17fd7d51862ba8e",
+            "placeholder": "​",
+            "style": "IPY_MODEL_d1f8f4568a444248b69022d58e3f1af0",
+            "value": "Batches: 100%"
+          }
+        },
+        "b0f8cf1f79e04b5fb47a810f2c81bd7e": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "b28d46c2ecdd46b9b3f2da871afbf1cb": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_0ac8e976a32c4f5989392b8088546e00",
+            "placeholder": "​",
+            "style": "IPY_MODEL_ed4b0035752546cc81688a7a77ba27c0",
+            "value": "Batches: 100%"
+          }
+        },
+        "b3eedd82e7da4ce8b3ded70e49a2afd0": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "FloatProgressModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "FloatProgressModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "ProgressView",
+            "bar_style": "success",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_36b5bc19b2d0407f8ab28ff0da2ce12d",
+            "max": 466247,
+            "min": 0,
+            "orientation": "horizontal",
+            "style": "IPY_MODEL_879e48d9a9e04183903d94ffe98313d2",
+            "value": 466247
+          }
+        },
+        "b67cbbf32f844a19b219be612d5038c9": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "b68e5097d2504d2cbd7e19aa1aac3a04": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "ProgressStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "ProgressStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "bar_color": null,
+            "description_width": ""
+          }
+        },
+        "b6a0eb553b024a71b737ff47ca8f7633": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "b7b7467ece304ffbbd352b9b96a03aad": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_d1e67c28b4664e8098dce8f5e80b8779",
+            "placeholder": "​",
+            "style": "IPY_MODEL_abe6cf39b784436993fcbe92221c31a3",
+            "value": " 90.9M/90.9M [00:00<00:00, 215MB/s]"
+          }
+        },
+        "b7e35038ce344110b785753b655130f5": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "ProgressStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "ProgressStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "bar_color": null,
+            "description_width": ""
+          }
+        },
+        "bb0d168c41f540b8ae42239d3938483a": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "FloatProgressModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "FloatProgressModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "ProgressView",
+            "bar_style": "success",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_67e37a088be64a2ba786ca923b1017dd",
+            "max": 10659,
+            "min": 0,
+            "orientation": "horizontal",
+            "style": "IPY_MODEL_98786f52ef5345b0b9164b9c1f2b8e18",
+            "value": 10659
+          }
+        },
+        "bda474c3b8184597a6a9bc6da0672a50": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "FloatProgressModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "FloatProgressModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "ProgressView",
+            "bar_style": "success",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_0c2e30d78c234b1b8098d879442d3bac",
+            "max": 1,
+            "min": 0,
+            "orientation": "horizontal",
+            "style": "IPY_MODEL_9bb8bf12010f42b2b17c10c7ccaa7bf8",
+            "value": 1
+          }
+        },
+        "bffc0f4b12f141398535990709fd4f2c": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_30798f87a8b848d783fdacd71af5dc04",
+            "placeholder": "​",
+            "style": "IPY_MODEL_07ce54c75e76488ba4019a20b3707061",
+            "value": "special_tokens_map.json: 100%"
+          }
+        },
+        "c690da8daa1e4f9ea73bcacdd92e8a6d": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "c83c23161674484e81f0db9856c23eb6": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_42335bcbc6ee40a79d36c5159cc7da06",
+            "placeholder": "​",
+            "style": "IPY_MODEL_cf694e1b797246b096ae588973dc985f",
+            "value": " 1/1 [00:00<00:00, 14.00it/s]"
+          }
+        },
+        "cf453a1ed54645aba656f9a3f1461e69": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "cf694e1b797246b096ae588973dc985f": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "cfcb6e456c354d99be91f161552f3376": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "ProgressStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "ProgressStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "bar_color": null,
+            "description_width": ""
+          }
+        },
+        "cfe6be8fd8254bc084a81b1d06e86ae1": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "d021a18ab70b4c7e8aec43932a124c36": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HBoxModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HBoxModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HBoxView",
+            "box_style": "",
+            "children": [
+              "IPY_MODEL_72e7c092fb054b7ea0dcd2782b5d8a7d",
+              "IPY_MODEL_8b1ea80221174fae943d5c9f997dfb57",
+              "IPY_MODEL_f8073d625f80415dbf712cee434f6e3a"
+            ],
+            "layout": "IPY_MODEL_5f6014ba13fa4a659b9eb1b5f83599a7"
+          }
+        },
+        "d032d1e7b4b54ba28ac83c1a12b23876": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "d0b161ae25c441e8b3caf7a3d88c1b05": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "d1e67c28b4664e8098dce8f5e80b8779": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "d1f8f4568a444248b69022d58e3f1af0": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "d2473b7a6c5b4483981516af2fc59bde": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "d25492ad867141bfa8d957d2464b8639": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "d452b32c54e14e41a17fd7d51862ba8e": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "d5c9977838a249eeab6ef628279b8155": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_61bd0d490c0e4c04a331cf9ce6b7d38f",
+            "placeholder": "​",
+            "style": "IPY_MODEL_7d8653fca29f4df3a7487733ff9db60b",
+            "value": " 116/116 [00:00<00:00, 5.06kB/s]"
+          }
+        },
+        "d9de065c7f81443e98ddf066c7b5bd54": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HBoxModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HBoxModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HBoxView",
+            "box_style": "",
+            "children": [
+              "IPY_MODEL_1e836106837c4ac7a11b36e700c46b64",
+              "IPY_MODEL_55591e8179084fcfa3a61c8bd8d09dcb",
+              "IPY_MODEL_de1ef93c41364eda9b4b111231057348"
+            ],
+            "layout": "IPY_MODEL_23b0b2f4f82c4a21846e91d7cea91da5"
+          }
+        },
+        "dd85d37dd1d14c7ea4592f8e11b2d2c8": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "de1ef93c41364eda9b4b111231057348": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_891cb726d45c4fef8f2c74a56df5532b",
+            "placeholder": "​",
+            "style": "IPY_MODEL_fa39189070334939aea5fa4a7de5ec8b",
+            "value": " 612/612 [00:00<00:00, 48.3kB/s]"
+          }
+        },
+        "e11f8c3891284e07bd2572257afd5e1b": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HBoxModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HBoxModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HBoxView",
+            "box_style": "",
+            "children": [
+              "IPY_MODEL_ee18d96394994d01b49d5b03b3d9a019",
+              "IPY_MODEL_844b06df5749441fab6f61656ce581a9",
+              "IPY_MODEL_e1c6b9a20e074f17aeba976b24e80c65"
+            ],
+            "layout": "IPY_MODEL_c690da8daa1e4f9ea73bcacdd92e8a6d"
+          }
+        },
+        "e1c6b9a20e074f17aeba976b24e80c65": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_22a665deff88477b9372c0350c4c572b",
+            "placeholder": "​",
+            "style": "IPY_MODEL_5e535ed2b83e496ab57b1c80b615ab0c",
+            "value": " 53.0/53.0 [00:00<00:00, 4.23kB/s]"
+          }
+        },
+        "e2e49c25d6fc4592b317e94cfabc2e5e": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "e61fdef1dc4b4d809168c0b441b0e6ac": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "e662ba10fbae49d9b66172125dfc0717": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "e82b5196209f4b9f919c7abb402a4504": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_d2473b7a6c5b4483981516af2fc59bde",
+            "placeholder": "​",
+            "style": "IPY_MODEL_4282ee7d947e426ba863df9970e82f3f",
+            "value": "Batches: 100%"
+          }
+        },
+        "ec747bd7c37c45298896c513634cd59a": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "ed4b0035752546cc81688a7a77ba27c0": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "edc4d84302f746d39a43e8107af6b67b": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HBoxModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HBoxModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HBoxView",
+            "box_style": "",
+            "children": [
+              "IPY_MODEL_980292182c7144e194604c13ac544a26",
+              "IPY_MODEL_8dee873065a047799a04e49ab791e449",
+              "IPY_MODEL_29683ef34d5646c687118a2a0cdec6d4"
+            ],
+            "layout": "IPY_MODEL_3ec694106303491ea112a257309bc69c"
+          }
+        },
+        "ee18d96394994d01b49d5b03b3d9a019": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_d0b161ae25c441e8b3caf7a3d88c1b05",
+            "placeholder": "​",
+            "style": "IPY_MODEL_47cf4b6b835d43388576a2abf4cc54f8",
+            "value": "sentence_bert_config.json: 100%"
+          }
+        },
+        "ef4f63fe9d8f4683a9d20becb6e4e2cb": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "f023175de68445f98a6b01bb40ccdc6d": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "f0e107dd6d54483aa367da0e337a97cd": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HBoxModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HBoxModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HBoxView",
+            "box_style": "",
+            "children": [
+              "IPY_MODEL_861a00796f55470e85d94733eeee9a5f",
+              "IPY_MODEL_5459633eb6e94ec391d13fcf67425726",
+              "IPY_MODEL_b7b7467ece304ffbbd352b9b96a03aad"
+            ],
+            "layout": "IPY_MODEL_9dece059f1204e29b106fca9e191ddb3"
+          }
+        },
+        "f5596c1c9c4d42f3bc171961f9582eff": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_4cf1dc345ace4da59f978f661487f975",
+            "placeholder": "​",
+            "style": "IPY_MODEL_8f30fca71bf24e5ca26e17c2321f893c",
+            "value": "1_Pooling/config.json: 100%"
+          }
+        },
+        "f6ecca7a1a8340fbbe056235a2714fc3": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "ProgressStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "ProgressStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "bar_color": null,
+            "description_width": ""
+          }
+        },
+        "f8073d625f80415dbf712cee434f6e3a": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "HTMLModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_541b9b4e74614e2cb855bb90f03df538",
+            "placeholder": "​",
+            "style": "IPY_MODEL_ff256b2275f740ed82bca4f43b4d6fd2",
+            "value": " 350/350 [00:00<00:00, 23.3kB/s]"
+          }
+        },
+        "f9b768c703494dd198f2978aff4892e8": {
+          "model_module": "@jupyter-widgets/base",
+          "model_module_version": "1.2.0",
+          "model_name": "LayoutModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "fa39189070334939aea5fa4a7de5ec8b": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "fbbcc19886cc43b38424fbb184162c61": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "FloatProgressModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "FloatProgressModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "ProgressView",
+            "bar_style": "success",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_754deb3970604d48a522bc9f021ad945",
+            "max": 349,
+            "min": 0,
+            "orientation": "horizontal",
+            "style": "IPY_MODEL_f6ecca7a1a8340fbbe056235a2714fc3",
+            "value": 349
+          }
+        },
+        "fe34706489c14253a5015ff6332ec4e0": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "FloatProgressModel",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "FloatProgressModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "ProgressView",
+            "bar_style": "success",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_cfe6be8fd8254bc084a81b1d06e86ae1",
+            "max": 1,
+            "min": 0,
+            "orientation": "horizontal",
+            "style": "IPY_MODEL_1817f6732a5f44c7adc75a644b1acef2",
+            "value": 1
+          }
+        },
+        "ff256b2275f740ed82bca4f43b4d6fd2": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "ff54451a48394faaaa9d8cdb690d0718": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_module_version": "1.5.0",
+          "model_name": "DescriptionStyleModel",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
         }
       }
     }
diff --git a/docs/notebooks/Llama_Stack_Benchmark_Evals.ipynb b/docs/notebooks/Llama_Stack_Benchmark_Evals.ipynb
index 4810425d2..61b5ab178 100644
--- a/docs/notebooks/Llama_Stack_Benchmark_Evals.ipynb
+++ b/docs/notebooks/Llama_Stack_Benchmark_Evals.ipynb
@@ -105,6 +105,7 @@
         }
       ],
       "source": [
+        "# NBVAL_SKIP\n",
         "!pip install -U llama-stack"
       ]
     },
@@ -309,12 +310,13 @@
         }
       ],
       "source": [
+        "# NBVAL_SKIP\n",
         "!llama stack build --template together --image-type venv"
       ]
     },
     {
       "cell_type": "code",
-      "execution_count": null,
+      "execution_count": 1,
       "metadata": {
         "colab": {
           "base_uri": "https://localhost:8080/"
@@ -328,7 +330,16 @@
           "name": "stdout",
           "output_type": "stream",
           "text": [
-            "Warning: `bwrap` is not available. Code interpreter tool will not work correctly.\n"
+            "Not in Google Colab environment\n",
+            "\u001b[33mWarning: `bwrap` is not available. Code interpreter tool will not work correctly.\u001b[0m\n"
+          ]
+        },
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "/opt/anaconda3/envs/master/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+            "  from .autonotebook import tqdm as notebook_tqdm\n"
           ]
         },
         {
@@ -356,63 +367,83 @@
               "- safety\n",
               "- scoring\n",
               "- telemetry\n",
-              "conda_env: together\n",
+              "- tool_runtime\n",
               "datasets: []\n",
-              "docker_image: null\n",
+              "container_image: null\n",
               "eval_tasks: []\n",
               "image_name: together\n",
               "memory_banks: []\n",
               "metadata_store:\n",
-              "  db_path: /root/.llama/distributions/together/registry.db\n",
+              "  db_path: /Users/xiyan/.llama/distributions/together/registry.db\n",
               "  namespace: null\n",
               "  type: sqlite\n",
               "models:\n",
               "- metadata: {}\n",
               "  model_id: meta-llama/Llama-3.1-8B-Instruct\n",
-              "  model_type: &id001 !!python/object/apply:llama_stack.apis.models.models.ModelType\n",
+              "  model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n",
               "  - llm\n",
-              "  provider_id: null\n",
+              "  provider_id: together\n",
               "  provider_model_id: meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo\n",
               "- metadata: {}\n",
               "  model_id: meta-llama/Llama-3.1-70B-Instruct\n",
-              "  model_type: *id001\n",
-              "  provider_id: null\n",
+              "  model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n",
+              "  - llm\n",
+              "  provider_id: together\n",
               "  provider_model_id: meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo\n",
               "- metadata: {}\n",
               "  model_id: meta-llama/Llama-3.1-405B-Instruct-FP8\n",
-              "  model_type: *id001\n",
-              "  provider_id: null\n",
+              "  model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n",
+              "  - llm\n",
+              "  provider_id: together\n",
               "  provider_model_id: meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo\n",
               "- metadata: {}\n",
               "  model_id: meta-llama/Llama-3.2-3B-Instruct\n",
-              "  model_type: *id001\n",
-              "  provider_id: null\n",
+              "  model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n",
+              "  - llm\n",
+              "  provider_id: together\n",
               "  provider_model_id: meta-llama/Llama-3.2-3B-Instruct-Turbo\n",
               "- metadata: {}\n",
               "  model_id: meta-llama/Llama-3.2-11B-Vision-Instruct\n",
-              "  model_type: *id001\n",
-              "  provider_id: null\n",
+              "  model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n",
+              "  - llm\n",
+              "  provider_id: together\n",
               "  provider_model_id: meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo\n",
               "- metadata: {}\n",
               "  model_id: meta-llama/Llama-3.2-90B-Vision-Instruct\n",
-              "  model_type: *id001\n",
-              "  provider_id: null\n",
+              "  model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n",
+              "  - llm\n",
+              "  provider_id: together\n",
               "  provider_model_id: meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo\n",
               "- metadata: {}\n",
+              "  model_id: meta-llama/Llama-3.3-70B-Instruct\n",
+              "  model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n",
+              "  - llm\n",
+              "  provider_id: together\n",
+              "  provider_model_id: meta-llama/Llama-3.3-70B-Instruct-Turbo\n",
+              "- metadata: {}\n",
               "  model_id: meta-llama/Llama-Guard-3-8B\n",
-              "  model_type: *id001\n",
-              "  provider_id: null\n",
+              "  model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n",
+              "  - llm\n",
+              "  provider_id: together\n",
               "  provider_model_id: meta-llama/Meta-Llama-Guard-3-8B\n",
               "- metadata: {}\n",
               "  model_id: meta-llama/Llama-Guard-3-11B-Vision\n",
-              "  model_type: *id001\n",
-              "  provider_id: null\n",
+              "  model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n",
+              "  - llm\n",
+              "  provider_id: together\n",
               "  provider_model_id: meta-llama/Llama-Guard-3-11B-Vision-Turbo\n",
+              "- metadata:\n",
+              "    embedding_dimension: 384\n",
+              "  model_id: all-MiniLM-L6-v2\n",
+              "  model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n",
+              "  - embedding\n",
+              "  provider_id: sentence-transformers\n",
+              "  provider_model_id: null\n",
               "providers:\n",
               "  agents:\n",
               "  - config:\n",
               "      persistence_store:\n",
-              "        db_path: /root/.llama/distributions/together/agents_store.db\n",
+              "        db_path: /Users/xiyan/.llama/distributions/together/agents_store.db\n",
               "        namespace: null\n",
               "        type: sqlite\n",
               "    provider_id: meta-reference\n",
@@ -430,14 +461,17 @@
               "    provider_type: inline::meta-reference\n",
               "  inference:\n",
               "  - config:\n",
-              "      api_key: 4985b03e627419b2964d34b8519ac6c4319f094d1ffb4f45514b4eb87e5427a2\n",
+              "      api_key: '********'\n",
               "      url: https://api.together.xyz/v1\n",
               "    provider_id: together\n",
               "    provider_type: remote::together\n",
+              "  - config: {}\n",
+              "    provider_id: sentence-transformers\n",
+              "    provider_type: inline::sentence-transformers\n",
               "  memory:\n",
               "  - config:\n",
               "      kvstore:\n",
-              "        db_path: /root/.llama/distributions/together/faiss_store.db\n",
+              "        db_path: /Users/xiyan/.llama/distributions/together/faiss_store.db\n",
               "        namespace: null\n",
               "        type: sqlite\n",
               "    provider_id: faiss\n",
@@ -454,22 +488,52 @@
               "    provider_id: llm-as-judge\n",
               "    provider_type: inline::llm-as-judge\n",
               "  - config:\n",
-              "      openai_api_key: ''\n",
+              "      openai_api_key: '********'\n",
               "    provider_id: braintrust\n",
               "    provider_type: inline::braintrust\n",
               "  telemetry:\n",
               "  - config:\n",
               "      service_name: llama-stack\n",
               "      sinks: sqlite\n",
-              "      sqlite_db_path: /root/.llama/distributions/together/trace_store.db\n",
+              "      sqlite_db_path: /Users/xiyan/.llama/distributions/together/trace_store.db\n",
               "    provider_id: meta-reference\n",
               "    provider_type: inline::meta-reference\n",
+              "  tool_runtime:\n",
+              "  - config:\n",
+              "      api_key: '********'\n",
+              "      max_results: 3\n",
+              "    provider_id: brave-search\n",
+              "    provider_type: remote::brave-search\n",
+              "  - config:\n",
+              "      api_key: '********'\n",
+              "      max_results: 3\n",
+              "    provider_id: tavily-search\n",
+              "    provider_type: remote::tavily-search\n",
+              "  - config: {}\n",
+              "    provider_id: code-interpreter\n",
+              "    provider_type: inline::code-interpreter\n",
+              "  - config: {}\n",
+              "    provider_id: rag-runtime\n",
+              "    provider_type: inline::rag-runtime\n",
               "scoring_fns: []\n",
               "shields:\n",
               "- params: null\n",
               "  provider_id: null\n",
               "  provider_shield_id: null\n",
               "  shield_id: meta-llama/Llama-Guard-3-8B\n",
+              "tool_groups:\n",
+              "- args: null\n",
+              "  mcp_endpoint: null\n",
+              "  provider_id: tavily-search\n",
+              "  toolgroup_id: builtin::websearch\n",
+              "- args: null\n",
+              "  mcp_endpoint: null\n",
+              "  provider_id: rag-runtime\n",
+              "  toolgroup_id: builtin::rag\n",
+              "- args: null\n",
+              "  mcp_endpoint: null\n",
+              "  provider_id: code-interpreter\n",
+              "  toolgroup_id: builtin::code_interpreter\n",
               "version: '2'\n",
               "\n",
               "
\n" @@ -484,63 +548,83 @@ "- safety\n", "- scoring\n", "- telemetry\n", - "conda_env: together\n", + "- tool_runtime\n", "datasets: \u001b[1m[\u001b[0m\u001b[1m]\u001b[0m\n", - "docker_image: null\n", + "container_image: null\n", "eval_tasks: \u001b[1m[\u001b[0m\u001b[1m]\u001b[0m\n", "image_name: together\n", "memory_banks: \u001b[1m[\u001b[0m\u001b[1m]\u001b[0m\n", "metadata_store:\n", - " db_path: \u001b[35m/root/.llama/distributions/together/\u001b[0m\u001b[95mregistry.db\u001b[0m\n", + " db_path: \u001b[35m/Users/xiyan/.llama/distributions/together/\u001b[0m\u001b[95mregistry.db\u001b[0m\n", " namespace: null\n", " type: sqlite\n", "models:\n", "- metadata: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", " model_id: meta-llama/Llama-\u001b[1;36m3.1\u001b[0m-8B-Instruct\n", - " model_type: &id001 !!python/object/apply:llama_stack.apis.models.models.ModelType\n", + " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", " - llm\n", - " provider_id: null\n", + " provider_id: together\n", " provider_model_id: meta-llama/Meta-Llama-\u001b[1;36m3.1\u001b[0m-8B-Instruct-Turbo\n", "- metadata: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", " model_id: meta-llama/Llama-\u001b[1;36m3.1\u001b[0m-70B-Instruct\n", - " model_type: *id001\n", - " provider_id: null\n", + " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", + " - llm\n", + " provider_id: together\n", " provider_model_id: meta-llama/Meta-Llama-\u001b[1;36m3.1\u001b[0m-70B-Instruct-Turbo\n", "- metadata: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", " model_id: meta-llama/Llama-\u001b[1;36m3.1\u001b[0m-405B-Instruct-FP8\n", - " model_type: *id001\n", - " provider_id: null\n", + " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", + " - llm\n", + " provider_id: together\n", " provider_model_id: meta-llama/Meta-Llama-\u001b[1;36m3.1\u001b[0m-405B-Instruct-Turbo\n", "- metadata: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", " model_id: meta-llama/Llama-\u001b[1;36m3.2\u001b[0m-3B-Instruct\n", - " model_type: *id001\n", - " provider_id: null\n", + " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", + " - llm\n", + " provider_id: together\n", " provider_model_id: meta-llama/Llama-\u001b[1;36m3.2\u001b[0m-3B-Instruct-Turbo\n", "- metadata: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", " model_id: meta-llama/Llama-\u001b[1;36m3.2\u001b[0m-11B-Vision-Instruct\n", - " model_type: *id001\n", - " provider_id: null\n", + " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", + " - llm\n", + " provider_id: together\n", " provider_model_id: meta-llama/Llama-\u001b[1;36m3.2\u001b[0m-11B-Vision-Instruct-Turbo\n", "- metadata: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", " model_id: meta-llama/Llama-\u001b[1;36m3.2\u001b[0m-90B-Vision-Instruct\n", - " model_type: *id001\n", - " provider_id: null\n", + " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", + " - llm\n", + " provider_id: together\n", " provider_model_id: meta-llama/Llama-\u001b[1;36m3.2\u001b[0m-90B-Vision-Instruct-Turbo\n", "- metadata: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", + " model_id: meta-llama/Llama-\u001b[1;36m3.3\u001b[0m-70B-Instruct\n", + " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", + " - llm\n", + " provider_id: together\n", + " provider_model_id: meta-llama/Llama-\u001b[1;36m3.3\u001b[0m-70B-Instruct-Turbo\n", + "- metadata: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", " model_id: meta-llama/Llama-Guard-\u001b[1;36m3\u001b[0m-8B\n", - " model_type: *id001\n", - " provider_id: null\n", + " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", + " - llm\n", + " provider_id: together\n", " provider_model_id: meta-llama/Meta-Llama-Guard-\u001b[1;36m3\u001b[0m-8B\n", "- metadata: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", " model_id: meta-llama/Llama-Guard-\u001b[1;36m3\u001b[0m-11B-Vision\n", - " model_type: *id001\n", - " provider_id: null\n", + " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", + " - llm\n", + " provider_id: together\n", " provider_model_id: meta-llama/Llama-Guard-\u001b[1;36m3\u001b[0m-11B-Vision-Turbo\n", + "- metadata:\n", + " embedding_dimension: \u001b[1;36m384\u001b[0m\n", + " model_id: all-MiniLM-L6-v2\n", + " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", + " - embedding\n", + " provider_id: sentence-transformers\n", + " provider_model_id: null\n", "providers:\n", " agents:\n", " - config:\n", " persistence_store:\n", - " db_path: \u001b[35m/root/.llama/distributions/together/\u001b[0m\u001b[95magents_store.db\u001b[0m\n", + " db_path: \u001b[35m/Users/xiyan/.llama/distributions/together/\u001b[0m\u001b[95magents_store.db\u001b[0m\n", " namespace: null\n", " type: sqlite\n", " provider_id: meta-reference\n", @@ -558,14 +642,17 @@ " provider_type: inline::meta-reference\n", " inference:\n", " - config:\n", - " api_key: 4985b03e627419b2964d34b8519ac6c4319f094d1ffb4f45514b4eb87e5427a2\n", + " api_key: \u001b[32m'********'\u001b[0m\n", " url: \u001b[4;94mhttps://api.together.xyz/v1\u001b[0m\n", " provider_id: together\n", " provider_type: remote::together\n", + " - config: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", + " provider_id: sentence-transformers\n", + " provider_type: inline::sentence-transformers\n", " memory:\n", " - config:\n", " kvstore:\n", - " db_path: \u001b[35m/root/.llama/distributions/together/\u001b[0m\u001b[95mfaiss_store.db\u001b[0m\n", + " db_path: \u001b[35m/Users/xiyan/.llama/distributions/together/\u001b[0m\u001b[95mfaiss_store.db\u001b[0m\n", " namespace: null\n", " type: sqlite\n", " provider_id: faiss\n", @@ -582,56 +669,74 @@ " provider_id: llm-as-judge\n", " provider_type: inline::llm-as-judge\n", " - config:\n", - " openai_api_key: \u001b[32m''\u001b[0m\n", + " openai_api_key: \u001b[32m'********'\u001b[0m\n", " provider_id: braintrust\n", " provider_type: inlin\u001b[1;92me::b\u001b[0mraintrust\n", " telemetry:\n", " - config:\n", " service_name: llama-stack\n", " sinks: sqlite\n", - " sqlite_db_path: \u001b[35m/root/.llama/distributions/together/\u001b[0m\u001b[95mtrace_store.db\u001b[0m\n", + " sqlite_db_path: \u001b[35m/Users/xiyan/.llama/distributions/together/\u001b[0m\u001b[95mtrace_store.db\u001b[0m\n", " provider_id: meta-reference\n", " provider_type: inline::meta-reference\n", + " tool_runtime:\n", + " - config:\n", + " api_key: \u001b[32m'********'\u001b[0m\n", + " max_results: \u001b[1;36m3\u001b[0m\n", + " provider_id: brave-search\n", + " provider_type: remot\u001b[1;92me::b\u001b[0mrave-search\n", + " - config:\n", + " api_key: \u001b[32m'********'\u001b[0m\n", + " max_results: \u001b[1;36m3\u001b[0m\n", + " provider_id: tavily-search\n", + " provider_type: remote::tavily-search\n", + " - config: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", + " provider_id: code-interpreter\n", + " provider_type: inlin\u001b[1;92me::c\u001b[0mode-interpreter\n", + " - config: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", + " provider_id: rag-runtime\n", + " provider_type: inline::rag-runtime\n", "scoring_fns: \u001b[1m[\u001b[0m\u001b[1m]\u001b[0m\n", "shields:\n", "- params: null\n", " provider_id: null\n", " provider_shield_id: null\n", " shield_id: meta-llama/Llama-Guard-\u001b[1;36m3\u001b[0m-8B\n", + "tool_groups:\n", + "- args: null\n", + " mcp_endpoint: null\n", + " provider_id: tavily-search\n", + " toolgroup_id: builtin::websearch\n", + "- args: null\n", + " mcp_endpoint: null\n", + " provider_id: rag-runtime\n", + " toolgroup_id: builtin::rag\n", + "- args: null\n", + " mcp_endpoint: null\n", + " provider_id: code-interpreter\n", + " toolgroup_id: builtin::code_interpreter\n", "version: \u001b[32m'2'\u001b[0m\n", "\n" ] }, "metadata": {}, "output_type": "display_data" - }, - { - "data": { - "text/plain": [ - "Model(identifier='meta-llama/Llama-3.1-405B-Instruct', metadata={}, provider_id='together', provider_resource_id='meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo', type='model', model_type='llm')" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" } ], "source": [ "import os\n", - "from google.colab import userdata\n", "\n", - "os.environ['TOGETHER_API_KEY'] = userdata.get('TOGETHER_API_KEY')\n", + "try:\n", + " from google.colab import userdata\n", + " os.environ['TOGETHER_API_KEY'] = userdata.get('TOGETHER_API_KEY')\n", + " os.environ['TAVILY_SEARCH_API_KEY'] = userdata.get('TAVILY_SEARCH_API_KEY')\n", + "except ImportError:\n", + " print(\"Not in Google Colab environment\")\n", "\n", "from llama_stack.distribution.library_client import LlamaStackAsLibraryClient\n", - "client = LlamaStackAsLibraryClient(\"together\")\n", - "_ = client.initialize()\n", "\n", - "# register 405B as LLM Judge model\n", - "client.models.register(\n", - " model_id=\"meta-llama/Llama-3.1-405B-Instruct\",\n", - " provider_model_id=\"meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo\",\n", - " provider_id=\"together\",\n", - ")" + "client = LlamaStackAsLibraryClient(\"together\")\n", + "_ = client.initialize()" ] }, { @@ -660,7 +765,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": { "id": "TC_IwIAQo4q-" }, @@ -673,7 +778,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": { "colab": { "base_uri": "https://localhost:8080/", @@ -786,137 +891,21 @@ }, "outputs": [ { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "feb82e061ee44283b4a46be858ef4cd7", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "README.md: 0%| | 0.00/36.0k [00:00EvaluateResponse(\n", "│ generations=[\n", + "│ │ {'generated_answer': 'Answer: D'},\n", "│ │ {\n", - "│ │ │ 'generated_answer': 'The Colorado potato beetle (Leptinotarsa decemlineata) is a significant pest of potatoes, causing damage to the leaves and stems of potato plants. The insect with black-colored antennae in the image is a Colorado potato beetle, which is known for its distinctive black and yellow stripes. On the other hand, the insect with tan-colored antennae is not a Colorado potato beetle and does not appear to be a pest of potatoes.\\n\\n*Answer*: B) The one with black coloured antennae'\n", + "│ │ │ 'generated_answer': 'The image shows a sunflower leaf with small, dark spots and white powdery patches. The dark spots are likely caused by a fungal pathogen, such as rust or septoria leaf spot, while the white powdery patches are likely caused by a fungal pathogen, such as powdery mildew.\\n\\nSince there are two distinct types of lesions on the leaf, it is likely that there are two different pathogens infecting the leaf.\\n\\n**Answer:** B) Two pathogens'\n", "│ │ },\n", "│ │ {\n", - "│ │ │ 'generated_answer': 'To determine the count of pathogens infecting this sunflower leaf, we need to analyze the image carefully. The image shows a sunflower leaf with several brown spots and patches on its surface. These brown spots and patches are indicative of fungal infections, which are common pathogens that affect sunflowers.\\n\\nUpon closer inspection, we can see that there are two distinct types of brown spots and patches on the leaf. One type is smaller and more circular in shape, while the other type is larger and more irregular in shape. This suggests that there may be two different pathogens infecting the leaf.\\n\\nHowever, without further information or testing, it is difficult to say for certain whether these two types of brown spots and patches are caused by different pathogens or if they are just different stages of the same infection. Therefore, based on the available information, the most likely answer is:\\n\\nAnswer: B) Two pathogens'\n", + "│ │ │ 'generated_answer': \"The question requires the identification of the reason behind the massive gum production on the trunks of grapefruit trees in Cyprus, despite appearing healthy from a distance. The correct answer can be deduced by analyzing the symptoms and considering the possible causes.\\n\\nTo determine the correct answer, let's evaluate each option:\\n\\nA) Don't know or not sure: This option is incorrect because it does not provide a specific reason for the gum production.\\n\\nB) Physiological stress: This option is also incorrect because it is too broad and does not specifically explain the gum production.\\n\\nC) Bacterial disease: This option is incorrect because bacterial diseases typically cause different symptoms such as leaf spots, blights, or wilting.\\n\\nD) Harvesting damage when cutting with knives: This option is incorrect because harvesting damage would likely cause wounds or scars on the tree, but it would not lead to massive gum production.\\n\\nE) Fungal gummosis: This option is the most likely cause of the gum production. Fungal gummosis is a common disease in citrus trees, including grapefruit, that causes the production of gum or sap on the trunks and branches. The disease is typically caused by fungi such as Phytophthora or Diplodia, which infect the tree through wounds or natural openings. The gum production is a defense mechanism by the tree to try to seal off the infection and prevent further damage.\\n\\nTherefore, the correct answer is:\\n\\nAnswer: E\"\n", "│ │ },\n", + "│ │ {'generated_answer': 'Answer: D'},\n", "│ │ {\n", - "│ │ │ 'generated_answer': 'Based on the image, the most likely reason for the massive gum production on the trunks of these grapefruit trees in Cyprus is a fungal infection. The gummosis, or the production of gum, is a common symptom of fungal diseases in citrus trees, and it can be caused by various factors such as root damage, water stress, or nutrient deficiencies. However, in this case, the presence of the gum on the trunks of the trees suggests that the cause is more likely related to a fungal infection.\\n\\nAnswer: E) Fungal gummosis'\n", - "│ │ },\n", - "│ │ {\n", - "│ │ │ 'generated_answer': 'The correct answer is D) Most viruses have a specific relationship with their vectors.\\n\\nExplanation:\\n\\n* Laboratory work with micro manipulators can mimic the transmission of viruses, but this is not the primary method of virus transmission in nature.\\n* Not all plant-feeding insects can transmit viruses; only specific species that have evolved to transmit particular viruses are capable of doing so.\\n* Similarly, not all plant viruses can be transmitted by insects; some are transmitted through other means such as mechanical transmission or nematodes.\\n* The correct assertion is that most viruses have a specific relationship with their vectors, meaning that each virus is typically transmitted by a specific type of insect or vector.\\n\\nAnswer: D'\n", - "│ │ },\n", - "│ │ {\n", - "│ │ │ 'generated_answer': \"The petioles of this rhubarb are splitting, and we need to determine which of the listed issues would not be the cause. \\n\\nFirst, let's consider physiological problems (A). Rhubarb is a hardy plant, but it can still experience physiological issues due to factors like temperature fluctuations, water stress, or nutrient deficiencies. These issues could potentially cause the petioles to split.\\n\\nNext, let's look at phytoplasma infection (B). Phytoplasmas are bacteria-like organisms that can infect plants, causing a range of symptoms including yellowing or browning of leaves, stunted growth, and distorted or split petioles. So, phytoplasma infection could also be a possible cause.\\n\\nNow, let's consider animal damage (D). Animals like rabbits, deer, or rodents might feed on the rhubarb leaves, causing damage to the petioles and potentially leading to splitting.\\n\\nFinally, let's think about bacteria (E). Bacterial infections can cause a range of symptoms in plants, including soft rot, leaf spot, and petiole splitting. So, bacteria could also be a potential cause.\\n\\nBased on this analysis, it seems that all of the listed issues could potentially cause the petioles of this rhubarb to split. Therefore, the correct answer is:\\n\\nAnswer: C\"\n", + "│ │ │ 'generated_answer': '**Causes of Splitting Petioles in Rhubarb**\\n\\nThe following factors can cause the petioles of rhubarb to split:\\n\\n* **Physiological Problems**: Issues such as water stress, nutrient deficiencies, or extreme temperatures can lead to splitting.\\n* **Phytoplasma Infection**: A bacterial infection caused by phytoplasma can lead to splitting of the petioles.\\n* **Animal Damage**: Pests like slugs, snails, or rodents can damage the plant and cause splitting.\\n* **Bacterial Infection**: Bacterial infections can also cause splitting.\\n\\nAs a result, the correct answer is:\\n\\n*Answer*: A) Physiological problems'\n", "│ │ }\n", "│ ],\n", "│ scores={\n", "│ │ 'basic::regex_parser_multiple_choice_answer': ScoringResult(\n", - "│ │ │ aggregated_results={'accuracy': 0.2, 'num_correct': 1.0, 'num_total': 5.0},\n", + "│ │ │ aggregated_results={'accuracy': {'accuracy': 0.2, 'num_correct': 1.0, 'num_total': 5}},\n", "│ │ │ score_rows=[{'score': 0.0}, {'score': 0.0}, {'score': 0.0}, {'score': 1.0}, {'score': 0.0}]\n", "│ │ )\n", "│ }\n", @@ -984,25 +969,21 @@ "text/plain": [ "\u001b[1;35mEvaluateResponse\u001b[0m\u001b[1m(\u001b[0m\n", "\u001b[2;32m│ \u001b[0m\u001b[33mgenerations\u001b[0m=\u001b[1m[\u001b[0m\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[1m{\u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m'Answer: D'\u001b[0m\u001b[1m}\u001b[0m,\n", "\u001b[2;32m│ │ \u001b[0m\u001b[1m{\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m'The Colorado potato beetle \u001b[0m\u001b[32m(\u001b[0m\u001b[32mLeptinotarsa decemlineata\u001b[0m\u001b[32m)\u001b[0m\u001b[32m is a significant pest of potatoes, causing damage to the leaves and stems of potato plants. The insect with black-colored antennae in the image is a Colorado potato beetle, which is known for its distinctive black and yellow stripes. On the other hand, the insect with tan-colored antennae is not a Colorado potato beetle and does not appear to be a pest of potatoes.\\n\\n*Answer*: B\u001b[0m\u001b[32m)\u001b[0m\u001b[32m The one with black coloured antennae'\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m'The image shows a sunflower leaf with small, dark spots and white powdery patches. The dark spots are likely caused by a fungal pathogen, such as rust or septoria leaf spot, while the white powdery patches are likely caused by a fungal pathogen, such as powdery mildew.\\n\\nSince there are two distinct types of lesions on the leaf, it is likely that there are two different pathogens infecting the leaf.\\n\\n**Answer:** B\u001b[0m\u001b[32m)\u001b[0m\u001b[32m Two pathogens'\u001b[0m\n", "\u001b[2;32m│ │ \u001b[0m\u001b[1m}\u001b[0m,\n", "\u001b[2;32m│ │ \u001b[0m\u001b[1m{\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m'To determine the count of pathogens infecting this sunflower leaf, we need to analyze the image carefully. The image shows a sunflower leaf with several brown spots and patches on its surface. These brown spots and patches are indicative of fungal infections, which are common pathogens that affect sunflowers.\\n\\nUpon closer inspection, we can see that there are two distinct types of brown spots and patches on the leaf. One type is smaller and more circular in shape, while the other type is larger and more irregular in shape. This suggests that there may be two different pathogens infecting the leaf.\\n\\nHowever, without further information or testing, it is difficult to say for certain whether these two types of brown spots and patches are caused by different pathogens or if they are just different stages of the same infection. Therefore, based on the available information, the most likely answer is:\\n\\nAnswer: B\u001b[0m\u001b[32m)\u001b[0m\u001b[32m Two pathogens'\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m\"The question requires the identification of the reason behind the massive gum production on the trunks of grapefruit trees in Cyprus, despite appearing healthy from a distance. The correct answer can be deduced by analyzing the symptoms and considering the possible causes.\\n\\nTo determine the correct answer, let's evaluate each option:\\n\\nA\u001b[0m\u001b[32m)\u001b[0m\u001b[32m Don't know or not sure: This option is incorrect because it does not provide a specific reason for the gum production.\\n\\nB\u001b[0m\u001b[32m)\u001b[0m\u001b[32m Physiological stress: This option is also incorrect because it is too broad and does not specifically explain the gum production.\\n\\nC\u001b[0m\u001b[32m)\u001b[0m\u001b[32m Bacterial disease: This option is incorrect because bacterial diseases typically cause different symptoms such as leaf spots, blights, or wilting.\\n\\nD\u001b[0m\u001b[32m)\u001b[0m\u001b[32m Harvesting damage when cutting with knives: This option is incorrect because harvesting damage would likely cause wounds or scars on the tree, but it would not lead to massive gum production.\\n\\nE\u001b[0m\u001b[32m)\u001b[0m\u001b[32m Fungal gummosis: This option is the most likely cause of the gum production. Fungal gummosis is a common disease in citrus trees, including grapefruit, that causes the production of gum or sap on the trunks and branches. The disease is typically caused by fungi such as Phytophthora or Diplodia, which infect the tree through wounds or natural openings. The gum production is a defense mechanism by the tree to try to seal off the infection and prevent further damage.\\n\\nTherefore, the correct answer is:\\n\\nAnswer: E\"\u001b[0m\n", "\u001b[2;32m│ │ \u001b[0m\u001b[1m}\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[1m{\u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m'Answer: D'\u001b[0m\u001b[1m}\u001b[0m,\n", "\u001b[2;32m│ │ \u001b[0m\u001b[1m{\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m'Based on the image, the most likely reason for the massive gum production on the trunks of these grapefruit trees in Cyprus is a fungal infection. The gummosis, or the production of gum, is a common symptom of fungal diseases in citrus trees, and it can be caused by various factors such as root damage, water stress, or nutrient deficiencies. However, in this case, the presence of the gum on the trunks of the trees suggests that the cause is more likely related to a fungal infection.\\n\\nAnswer: E\u001b[0m\u001b[32m)\u001b[0m\u001b[32m Fungal gummosis'\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[1m}\u001b[0m,\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[1m{\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m'The correct answer is D\u001b[0m\u001b[32m)\u001b[0m\u001b[32m Most viruses have a specific relationship with their vectors.\\n\\nExplanation:\\n\\n* Laboratory work with micro manipulators can mimic the transmission of viruses, but this is not the primary method of virus transmission in nature.\\n* Not all plant-feeding insects can transmit viruses; only specific species that have evolved to transmit particular viruses are capable of doing so.\\n* Similarly, not all plant viruses can be transmitted by insects; some are transmitted through other means such as mechanical transmission or nematodes.\\n* The correct assertion is that most viruses have a specific relationship with their vectors, meaning that each virus is typically transmitted by a specific type of insect or vector.\\n\\nAnswer: D'\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[1m}\u001b[0m,\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[1m{\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m\"The petioles of this rhubarb are splitting, and we need to determine which of the listed issues would not be the cause. \\n\\nFirst, let's consider physiological problems \u001b[0m\u001b[32m(\u001b[0m\u001b[32mA\u001b[0m\u001b[32m)\u001b[0m\u001b[32m. Rhubarb is a hardy plant, but it can still experience physiological issues due to factors like temperature fluctuations, water stress, or nutrient deficiencies. These issues could potentially cause the petioles to split.\\n\\nNext, let's look at phytoplasma infection \u001b[0m\u001b[32m(\u001b[0m\u001b[32mB\u001b[0m\u001b[32m)\u001b[0m\u001b[32m. Phytoplasmas are bacteria-like organisms that can infect plants, causing a range of symptoms including yellowing or browning of leaves, stunted growth, and distorted or split petioles. So, phytoplasma infection could also be a possible cause.\\n\\nNow, let's consider animal damage \u001b[0m\u001b[32m(\u001b[0m\u001b[32mD\u001b[0m\u001b[32m)\u001b[0m\u001b[32m. Animals like rabbits, deer, or rodents might feed on the rhubarb leaves, causing damage to the petioles and potentially leading to splitting.\\n\\nFinally, let's think about bacteria \u001b[0m\u001b[32m(\u001b[0m\u001b[32mE\u001b[0m\u001b[32m)\u001b[0m\u001b[32m. Bacterial infections can cause a range of symptoms in plants, including soft rot, leaf spot, and petiole splitting. So, bacteria could also be a potential cause.\\n\\nBased on this analysis, it seems that all of the listed issues could potentially cause the petioles of this rhubarb to split. Therefore, the correct answer is:\\n\\nAnswer: C\"\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m'**Causes of Splitting Petioles in Rhubarb**\\n\\nThe following factors can cause the petioles of rhubarb to split:\\n\\n* **Physiological Problems**: Issues such as water stress, nutrient deficiencies, or extreme temperatures can lead to splitting.\\n* **Phytoplasma Infection**: A bacterial infection caused by phytoplasma can lead to splitting of the petioles.\\n* **Animal Damage**: Pests like slugs, snails, or rodents can damage the plant and cause splitting.\\n* **Bacterial Infection**: Bacterial infections can also cause splitting.\\n\\nAs a result, the correct answer is:\\n\\n*Answer*: A\u001b[0m\u001b[32m)\u001b[0m\u001b[32m Physiological problems'\u001b[0m\n", "\u001b[2;32m│ │ \u001b[0m\u001b[1m}\u001b[0m\n", "\u001b[2;32m│ \u001b[0m\u001b[1m]\u001b[0m,\n", "\u001b[2;32m│ \u001b[0m\u001b[33mscores\u001b[0m=\u001b[1m{\u001b[0m\n", "\u001b[2;32m│ │ \u001b[0m\u001b[32m'basic::regex_parser_multiple_choice_answer'\u001b[0m: \u001b[1;35mScoringResult\u001b[0m\u001b[1m(\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[33maggregated_results\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'accuracy'\u001b[0m: \u001b[1;36m0.2\u001b[0m, \u001b[32m'num_correct'\u001b[0m: \u001b[1;36m1.0\u001b[0m, \u001b[32m'num_total'\u001b[0m: \u001b[1;36m5.0\u001b[0m\u001b[1m}\u001b[0m,\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[33maggregated_results\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'accuracy'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'accuracy'\u001b[0m: \u001b[1;36m0.2\u001b[0m, \u001b[32m'num_correct'\u001b[0m: \u001b[1;36m1.0\u001b[0m, \u001b[32m'num_total'\u001b[0m: \u001b[1;36m5\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n", "\u001b[2;32m│ │ │ \u001b[0m\u001b[33mscore_rows\u001b[0m=\u001b[1m[\u001b[0m\u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m, \u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m, \u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m, \u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[1;36m1.0\u001b[0m\u001b[1m}\u001b[0m, \u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m\u001b[1m]\u001b[0m\n", "\u001b[2;32m│ │ \u001b[0m\u001b[1m)\u001b[0m\n", "\u001b[2;32m│ \u001b[0m\u001b[1m}\u001b[0m\n", @@ -1014,8 +995,8 @@ } ], "source": [ - "from tqdm import tqdm\n", "from rich.pretty import pprint\n", + "from tqdm import tqdm\n", "\n", "SYSTEM_PROMPT_TEMPLATE = \"\"\"\n", "You are an expert in {subject} whose job is to answer questions from the user using images.\n", @@ -1039,7 +1020,7 @@ "client.eval_tasks.register(\n", " eval_task_id=\"meta-reference::mmmu\",\n", " dataset_id=f\"mmmu-{subset}-{split}\",\n", - " scoring_functions=[\"basic::regex_parser_multiple_choice_answer\"]\n", + " scoring_functions=[\"basic::regex_parser_multiple_choice_answer\"],\n", ")\n", "\n", "response = client.eval.evaluate_rows(\n", @@ -1052,16 +1033,19 @@ " \"type\": \"model\",\n", " \"model\": \"meta-llama/Llama-3.2-90B-Vision-Instruct\",\n", " \"sampling_params\": {\n", - " \"temperature\": 0.0,\n", + " \"strategy\": {\n", + " \"type\": \"top_p\",\n", + " \"temperature\": 1.0,\n", + " \"top_p\": 0.95,\n", + " },\n", " \"max_tokens\": 4096,\n", - " \"top_p\": 0.9,\n", " \"repeat_penalty\": 1.0,\n", " },\n", - " \"system_message\": system_message\n", - " }\n", - " }\n", + " \"system_message\": system_message,\n", + " },\n", + " },\n", ")\n", - "pprint(response)" + "pprint(response)\n" ] }, { @@ -1077,7 +1061,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": { "id": "HXmZf3Ymw-aX" }, @@ -1098,13 +1082,13 @@ " \"input_query\": {\"type\": \"string\"},\n", " \"expected_answer\": {\"type\": \"string\"},\n", " \"chat_completion_input\": {\"type\": \"chat_completion_input\"},\n", - " }\n", - ")" + " },\n", + ")\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": { "id": "Gc8azb4Rxr5J" }, @@ -1113,12 +1097,12 @@ "eval_rows = client.datasetio.get_rows_paginated(\n", " dataset_id=simpleqa_dataset_id,\n", " rows_in_page=5,\n", - ")" + ")\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": { "colab": { "base_uri": "https://localhost:8080/", @@ -1132,7 +1116,14 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 5/5 [00:48<00:00, 9.68s/it]\n" + " 0%| | 0/5 [00:00EvaluateResponse(\n", "│ generations=[\n", - "│ │ {'generated_answer': 'The recipient of the IEEE Frank Rosenblatt Award in 2010 was Vladimir Vapnik'},\n", + "│ │ {'generated_answer': \"I'm not sure who received the IEEE Frank Rosenblatt Award in 2010.\"},\n", + "│ │ {'generated_answer': \"I'm not aware of the information about the 2018 Jerlov Award recipient.\"},\n", "│ │ {\n", - "│ │ │ 'generated_answer': \"I am unable to verify who was awarded the Oceanography Society's Jerlov Award in 2018.\"\n", + "│ │ │ 'generated_answer': \"Radcliffe College was a women's liberal arts college in Cambridge, Massachusetts. However, it merged with Harvard University in 1977 and is now known as the Radcliffe Institute for Advanced Study at Harvard University.\"\n", "│ │ },\n", + "│ │ {'generated_answer': 'I do not have information on the Leipzig 1877 tournament.'},\n", "│ │ {\n", - "│ │ │ 'generated_answer': \"Radcliffe College was a women's liberal arts college, but it has since been integrated into Harvard University.\"\n", - "│ │ },\n", - "│ │ {\n", - "│ │ │ 'generated_answer': \"The Leipzig 1877 tournament was organized in the honor of 50th anniversary of the first chess club in Germany (the Leipzig Chess Club's) founding and of the 50th anniversary of Paul Morphy's birth\"\n", - "│ │ },\n", - "│ │ {\n", - "│ │ │ 'generated_answer': \"Karl Küchler's 1908 guidebook states that Empress Elizabeth of Austria's favorite sculpture, which was made for her villa Achilleion at Corfu, depicted 'Dying Achilles'.\"\n", + "│ │ │ 'generated_answer': \"I am unable to verify what Empress Elizabeth of Austria's favorite sculpture depicted at her villa Achilleion at Corfu, according to Karl Küchler.\"\n", "│ │ }\n", "│ ],\n", "│ scores={\n", "│ │ 'llm-as-judge::405b-simpleqa': ScoringResult(\n", "│ │ │ aggregated_results={},\n", "│ │ │ score_rows=[\n", - "│ │ │ │ {'score': 'B', 'judge_feedback': 'B'},\n", + "│ │ │ │ {'score': 'C', 'judge_feedback': 'C'},\n", "│ │ │ │ {'score': 'C', 'judge_feedback': 'C'},\n", "│ │ │ │ {'score': 'A', 'judge_feedback': 'A'},\n", - "│ │ │ │ {'score': 'B', 'judge_feedback': 'B'},\n", - "│ │ │ │ {'score': 'B', 'judge_feedback': 'B'}\n", + "│ │ │ │ {'score': 'C', 'judge_feedback': 'C'},\n", + "│ │ │ │ {'score': 'C', 'judge_feedback': 'C'}\n", "│ │ │ ]\n", "│ │ )\n", "│ }\n", @@ -1172,29 +1159,25 @@ "text/plain": [ "\u001b[1;35mEvaluateResponse\u001b[0m\u001b[1m(\u001b[0m\n", "\u001b[2;32m│ \u001b[0m\u001b[33mgenerations\u001b[0m=\u001b[1m[\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[1m{\u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m'The recipient of the IEEE Frank Rosenblatt Award in 2010 was Vladimir Vapnik'\u001b[0m\u001b[1m}\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[1m{\u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m\"I'm not sure who received the IEEE Frank Rosenblatt Award in 2010.\"\u001b[0m\u001b[1m}\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[1m{\u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m\"I'm not aware of the information about the 2018 Jerlov Award recipient.\"\u001b[0m\u001b[1m}\u001b[0m,\n", "\u001b[2;32m│ │ \u001b[0m\u001b[1m{\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m\"I am unable to verify who was awarded the Oceanography Society's Jerlov Award in 2018.\"\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m\"Radcliffe College was a women's liberal arts college in Cambridge, Massachusetts. However, it merged with Harvard University in 1977 and is now known as the Radcliffe Institute for Advanced Study at Harvard University.\"\u001b[0m\n", "\u001b[2;32m│ │ \u001b[0m\u001b[1m}\u001b[0m,\n", + "\u001b[2;32m│ │ \u001b[0m\u001b[1m{\u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m'I do not have information on the Leipzig 1877 tournament.'\u001b[0m\u001b[1m}\u001b[0m,\n", "\u001b[2;32m│ │ \u001b[0m\u001b[1m{\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m\"Radcliffe College was a women's liberal arts college, but it has since been integrated into Harvard University.\"\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[1m}\u001b[0m,\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[1m{\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m\"The Leipzig 1877 tournament was organized in the honor of 50th anniversary of the first chess club in Germany \u001b[0m\u001b[32m(\u001b[0m\u001b[32mthe Leipzig Chess Club's\u001b[0m\u001b[32m)\u001b[0m\u001b[32m founding and of the 50th anniversary of Paul Morphy's birth\"\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[1m}\u001b[0m,\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[1m{\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m\"Karl Küchler's 1908 guidebook states that Empress Elizabeth of Austria's favorite sculpture, which was made for her villa Achilleion at Corfu, depicted 'Dying Achilles'.\"\u001b[0m\n", + "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m\"I am unable to verify what Empress Elizabeth of Austria's favorite sculpture depicted at her villa Achilleion at Corfu, according to Karl Küchler.\"\u001b[0m\n", "\u001b[2;32m│ │ \u001b[0m\u001b[1m}\u001b[0m\n", "\u001b[2;32m│ \u001b[0m\u001b[1m]\u001b[0m,\n", "\u001b[2;32m│ \u001b[0m\u001b[33mscores\u001b[0m=\u001b[1m{\u001b[0m\n", "\u001b[2;32m│ │ \u001b[0m\u001b[32m'llm-as-judge::405b-simpleqa'\u001b[0m: \u001b[1;35mScoringResult\u001b[0m\u001b[1m(\u001b[0m\n", "\u001b[2;32m│ │ │ \u001b[0m\u001b[33maggregated_results\u001b[0m=\u001b[1m{\u001b[0m\u001b[1m}\u001b[0m,\n", "\u001b[2;32m│ │ │ \u001b[0m\u001b[33mscore_rows\u001b[0m=\u001b[1m[\u001b[0m\n", - "\u001b[2;32m│ │ │ │ \u001b[0m\u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[32m'B'\u001b[0m, \u001b[32m'judge_feedback'\u001b[0m: \u001b[32m'B'\u001b[0m\u001b[1m}\u001b[0m,\n", + "\u001b[2;32m│ │ │ │ \u001b[0m\u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[32m'C'\u001b[0m, \u001b[32m'judge_feedback'\u001b[0m: \u001b[32m'C'\u001b[0m\u001b[1m}\u001b[0m,\n", "\u001b[2;32m│ │ │ │ \u001b[0m\u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[32m'C'\u001b[0m, \u001b[32m'judge_feedback'\u001b[0m: \u001b[32m'C'\u001b[0m\u001b[1m}\u001b[0m,\n", "\u001b[2;32m│ │ │ │ \u001b[0m\u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[32m'A'\u001b[0m, \u001b[32m'judge_feedback'\u001b[0m: \u001b[32m'A'\u001b[0m\u001b[1m}\u001b[0m,\n", - "\u001b[2;32m│ │ │ │ \u001b[0m\u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[32m'B'\u001b[0m, \u001b[32m'judge_feedback'\u001b[0m: \u001b[32m'B'\u001b[0m\u001b[1m}\u001b[0m,\n", - "\u001b[2;32m│ │ │ │ \u001b[0m\u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[32m'B'\u001b[0m, \u001b[32m'judge_feedback'\u001b[0m: \u001b[32m'B'\u001b[0m\u001b[1m}\u001b[0m\n", + "\u001b[2;32m│ │ │ │ \u001b[0m\u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[32m'C'\u001b[0m, \u001b[32m'judge_feedback'\u001b[0m: \u001b[32m'C'\u001b[0m\u001b[1m}\u001b[0m,\n", + "\u001b[2;32m│ │ │ │ \u001b[0m\u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[32m'C'\u001b[0m, \u001b[32m'judge_feedback'\u001b[0m: \u001b[32m'C'\u001b[0m\u001b[1m}\u001b[0m\n", "\u001b[2;32m│ │ │ \u001b[0m\u001b[1m]\u001b[0m\n", "\u001b[2;32m│ │ \u001b[0m\u001b[1m)\u001b[0m\n", "\u001b[2;32m│ \u001b[0m\u001b[1m}\u001b[0m\n", @@ -1206,10 +1189,17 @@ } ], "source": [ + "# register 405B as LLM Judge model\n", + "client.models.register(\n", + " model_id=\"meta-llama/Llama-3.1-405B-Instruct\",\n", + " provider_model_id=\"meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo\",\n", + " provider_id=\"together\",\n", + ")\n", + "\n", "client.eval_tasks.register(\n", " eval_task_id=\"meta-reference::simpleqa\",\n", " dataset_id=simpleqa_dataset_id,\n", - " scoring_functions=[\"llm-as-judge::405b-simpleqa\"]\n", + " scoring_functions=[\"llm-as-judge::405b-simpleqa\"],\n", ")\n", "\n", "response = client.eval.evaluate_rows(\n", @@ -1222,15 +1212,16 @@ " \"type\": \"model\",\n", " \"model\": \"meta-llama/Llama-3.2-90B-Vision-Instruct\",\n", " \"sampling_params\": {\n", - " \"temperature\": 0.0,\n", + " \"strategy\": {\n", + " \"type\": \"greedy\",\n", + " },\n", " \"max_tokens\": 4096,\n", - " \"top_p\": 0.9,\n", " \"repeat_penalty\": 1.0,\n", " },\n", - " }\n", - " }\n", + " },\n", + " },\n", ")\n", - "pprint(response)" + "pprint(response)\n" ] }, { @@ -1252,7 +1243,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "metadata": { "colab": { "base_uri": "https://localhost:8080/", @@ -1266,7 +1257,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "5it [00:26, 5.29s/it]\n" + "5it [00:06, 1.33s/it]\n" ] }, { @@ -1275,27 +1266,25 @@ "
EvaluateResponse(\n",
               "│   generations=[\n",
               "│   │   {\n",
-              "│   │   │   'generated_answer': \"I'm sorry but I cannot find the recipient of the IEEE Frank Rosenblatt Award in 2010.\"\n",
+              "│   │   │   'generated_answer': 'The IEEE Frank Rosenblatt Award was given to Professor John Shawe-Taylor in 2010 for his contributions to the foundations of kernel methods.'\n",
               "│   │   },\n",
               "│   │   {\n",
-              "│   │   │   'generated_answer': \"I'm not sure who was awarded the Oceanography Society's Jerlov Award in 2018. Let me search for the information.\"\n",
+              "│   │   │   'generated_answer': 'The Jerlov Award is given by The Oceanography Society to recognize outstanding contributions to the field of ocean optics. The 2018 Jerlov Award was awarded to Dr. Kendall L. Carder.'\n",
               "│   │   },\n",
               "│   │   {\n",
-              "│   │   │   'generated_answer': \"The women's liberal arts college in Cambridge, Massachusetts is called Radcliffe College. However, in 1999, it merged with Harvard University and is now known as the Radcliffe Institute for Advanced Study at Harvard University.\"\n",
+              "│   │   │   'generated_answer': \"The women's liberal arts college in Cambridge, Massachusetts is Radcliffe College. However, in 1999, Radcliffe College merged with Harvard University to form the Radcliffe Institute for Advanced Study at Harvard University. The institute is still located in Cambridge, Massachusetts, and is dedicated to supporting women's education and research.\"\n",
               "│   │   },\n",
+              "│   │   {'generated_answer': 'The Leipzig 1877 tournament was organized in honor of Adolf Anderssen.'},\n",
               "│   │   {\n",
-              "│   │   │   'generated_answer': 'The 1877 Leipzig tournament was organized in honor of Anderssen, a German chess master.'\n",
-              "│   │   },\n",
-              "│   │   {\n",
-              "│   │   │   'generated_answer': \"Empress Elizabeth of Austria's favorite sculpture, made for her villa Achilleion at Corfu, depicted Achilles.\"\n",
+              "│   │   │   'generated_answer': \"According to Karl Küchler, Empress Elizabeth of Austria's favorite sculpture, which was made for her villa Achilleion at Corfu, depicted the Dying Achilles.\"\n",
               "│   │   }\n",
               "│   ],\n",
               "│   scores={\n",
               "│   │   'llm-as-judge::405b-simpleqa': ScoringResult(\n",
               "│   │   │   aggregated_results={},\n",
               "│   │   │   score_rows=[\n",
-              "│   │   │   │   {'score': 'C', 'judge_feedback': 'C.'},\n",
-              "│   │   │   │   {'score': 'C', 'judge_feedback': 'C'},\n",
+              "│   │   │   │   {'score': 'B', 'judge_feedback': 'B'},\n",
+              "│   │   │   │   {'score': 'B', 'judge_feedback': 'B'},\n",
               "│   │   │   │   {'score': 'A', 'judge_feedback': 'A'},\n",
               "│   │   │   │   {'score': 'A', 'judge_feedback': 'A'},\n",
               "│   │   │   │   {'score': 'B', 'judge_feedback': 'B'}\n",
@@ -1309,27 +1298,25 @@
               "\u001b[1;35mEvaluateResponse\u001b[0m\u001b[1m(\u001b[0m\n",
               "\u001b[2;32m│   \u001b[0m\u001b[33mgenerations\u001b[0m=\u001b[1m[\u001b[0m\n",
               "\u001b[2;32m│   │   \u001b[0m\u001b[1m{\u001b[0m\n",
-              "\u001b[2;32m│   │   │   \u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m\"I'm sorry but I cannot find the recipient of the IEEE Frank Rosenblatt Award in 2010.\"\u001b[0m\n",
+              "\u001b[2;32m│   │   │   \u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m'The IEEE Frank Rosenblatt Award was given to Professor John Shawe-Taylor in 2010 for his contributions to the foundations of kernel methods.'\u001b[0m\n",
               "\u001b[2;32m│   │   \u001b[0m\u001b[1m}\u001b[0m,\n",
               "\u001b[2;32m│   │   \u001b[0m\u001b[1m{\u001b[0m\n",
-              "\u001b[2;32m│   │   │   \u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m\"I'm not sure who was awarded the Oceanography Society's Jerlov Award in 2018. Let me search for the information.\"\u001b[0m\n",
+              "\u001b[2;32m│   │   │   \u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m'The Jerlov Award is given by The Oceanography Society to recognize outstanding contributions to the field of ocean optics. The 2018 Jerlov Award was awarded to Dr. Kendall L. Carder.'\u001b[0m\n",
               "\u001b[2;32m│   │   \u001b[0m\u001b[1m}\u001b[0m,\n",
               "\u001b[2;32m│   │   \u001b[0m\u001b[1m{\u001b[0m\n",
-              "\u001b[2;32m│   │   │   \u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m\"The women's liberal arts college in Cambridge, Massachusetts is called Radcliffe College. However, in 1999, it merged with Harvard University and is now known as the Radcliffe Institute for Advanced Study at Harvard University.\"\u001b[0m\n",
+              "\u001b[2;32m│   │   │   \u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m\"The women's liberal arts college in Cambridge, Massachusetts is Radcliffe College. However, in 1999, Radcliffe College merged with Harvard University to form the Radcliffe Institute for Advanced Study at Harvard University. The institute is still located in Cambridge, Massachusetts, and is dedicated to supporting women's education and research.\"\u001b[0m\n",
               "\u001b[2;32m│   │   \u001b[0m\u001b[1m}\u001b[0m,\n",
+              "\u001b[2;32m│   │   \u001b[0m\u001b[1m{\u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m'The Leipzig 1877 tournament was organized in honor of Adolf Anderssen.'\u001b[0m\u001b[1m}\u001b[0m,\n",
               "\u001b[2;32m│   │   \u001b[0m\u001b[1m{\u001b[0m\n",
-              "\u001b[2;32m│   │   │   \u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m'The 1877 Leipzig tournament was organized in honor of Anderssen, a German chess master.'\u001b[0m\n",
-              "\u001b[2;32m│   │   \u001b[0m\u001b[1m}\u001b[0m,\n",
-              "\u001b[2;32m│   │   \u001b[0m\u001b[1m{\u001b[0m\n",
-              "\u001b[2;32m│   │   │   \u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m\"Empress Elizabeth of Austria's favorite sculpture, made for her villa Achilleion at Corfu, depicted Achilles.\"\u001b[0m\n",
+              "\u001b[2;32m│   │   │   \u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m\"According to Karl Küchler, Empress Elizabeth of Austria's favorite sculpture, which was made for her villa Achilleion at Corfu, depicted the Dying Achilles.\"\u001b[0m\n",
               "\u001b[2;32m│   │   \u001b[0m\u001b[1m}\u001b[0m\n",
               "\u001b[2;32m│   \u001b[0m\u001b[1m]\u001b[0m,\n",
               "\u001b[2;32m│   \u001b[0m\u001b[33mscores\u001b[0m=\u001b[1m{\u001b[0m\n",
               "\u001b[2;32m│   │   \u001b[0m\u001b[32m'llm-as-judge::405b-simpleqa'\u001b[0m: \u001b[1;35mScoringResult\u001b[0m\u001b[1m(\u001b[0m\n",
               "\u001b[2;32m│   │   │   \u001b[0m\u001b[33maggregated_results\u001b[0m=\u001b[1m{\u001b[0m\u001b[1m}\u001b[0m,\n",
               "\u001b[2;32m│   │   │   \u001b[0m\u001b[33mscore_rows\u001b[0m=\u001b[1m[\u001b[0m\n",
-              "\u001b[2;32m│   │   │   │   \u001b[0m\u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[32m'C'\u001b[0m, \u001b[32m'judge_feedback'\u001b[0m: \u001b[32m'C.'\u001b[0m\u001b[1m}\u001b[0m,\n",
-              "\u001b[2;32m│   │   │   │   \u001b[0m\u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[32m'C'\u001b[0m, \u001b[32m'judge_feedback'\u001b[0m: \u001b[32m'C'\u001b[0m\u001b[1m}\u001b[0m,\n",
+              "\u001b[2;32m│   │   │   │   \u001b[0m\u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[32m'B'\u001b[0m, \u001b[32m'judge_feedback'\u001b[0m: \u001b[32m'B'\u001b[0m\u001b[1m}\u001b[0m,\n",
+              "\u001b[2;32m│   │   │   │   \u001b[0m\u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[32m'B'\u001b[0m, \u001b[32m'judge_feedback'\u001b[0m: \u001b[32m'B'\u001b[0m\u001b[1m}\u001b[0m,\n",
               "\u001b[2;32m│   │   │   │   \u001b[0m\u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[32m'A'\u001b[0m, \u001b[32m'judge_feedback'\u001b[0m: \u001b[32m'A'\u001b[0m\u001b[1m}\u001b[0m,\n",
               "\u001b[2;32m│   │   │   │   \u001b[0m\u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[32m'A'\u001b[0m, \u001b[32m'judge_feedback'\u001b[0m: \u001b[32m'A'\u001b[0m\u001b[1m}\u001b[0m,\n",
               "\u001b[2;32m│   │   │   │   \u001b[0m\u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[32m'B'\u001b[0m, \u001b[32m'judge_feedback'\u001b[0m: \u001b[32m'B'\u001b[0m\u001b[1m}\u001b[0m\n",
@@ -1345,25 +1332,23 @@
       ],
       "source": [
         "agent_config = {\n",
-        "    \"model\": \"meta-llama/Llama-3.1-405B-Instruct\",\n",
-        "    \"instructions\": \"You are a helpful assistant\",\n",
+        "    \"model\": \"meta-llama/Llama-3.3-70B-Instruct\",\n",
+        "    \"instructions\": \"You are a helpful assistant that have access to tool to search the web. \",\n",
         "    \"sampling_params\": {\n",
-        "        \"strategy\": \"greedy\",\n",
-        "        \"temperature\": 0.0,\n",
-        "        \"top_p\": 0.95,\n",
-        "    },\n",
-        "    \"tools\": [\n",
-        "        {\n",
-        "            \"type\": \"brave_search\",\n",
-        "            \"engine\": \"tavily\",\n",
-        "            \"api_key\": userdata.get(\"TAVILY_SEARCH_API_KEY\")\n",
+        "        \"strategy\": {\n",
+        "            \"type\": \"top_p\",\n",
+        "            \"temperature\": 0.5,\n",
+        "            \"top_p\": 0.9,\n",
         "        }\n",
+        "    },\n",
+        "    \"toolgroups\": [\n",
+        "        \"builtin::websearch\",\n",
         "    ],\n",
         "    \"tool_choice\": \"auto\",\n",
         "    \"tool_prompt_format\": \"json\",\n",
         "    \"input_shields\": [],\n",
         "    \"output_shields\": [],\n",
-        "    \"enable_session_persistence\": False\n",
+        "    \"enable_session_persistence\": False,\n",
         "}\n",
         "\n",
         "response = client.eval.evaluate_rows(\n",
@@ -1375,11 +1360,18 @@
         "        \"eval_candidate\": {\n",
         "            \"type\": \"agent\",\n",
         "            \"config\": agent_config,\n",
-        "        }\n",
-        "    }\n",
+        "        },\n",
+        "    },\n",
         ")\n",
-        "pprint(response)"
+        "pprint(response)\n"
       ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": []
     }
   ],
   "metadata": {
@@ -1395,7 +1387,16 @@
       "name": "python3"
     },
     "language_info": {
-      "name": "python"
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.10.16"
     },
     "widgets": {
       "application/vnd.jupyter.widget-state+json": {
diff --git a/docs/notebooks/Llama_Stack_Building_AI_Applications.ipynb b/docs/notebooks/Llama_Stack_Building_AI_Applications.ipynb
deleted file mode 100644
index 7e6284628..000000000
--- a/docs/notebooks/Llama_Stack_Building_AI_Applications.ipynb
+++ /dev/null
@@ -1,8570 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "markdown",
-      "id": "c1e7571c",
-      "metadata": {
-        "id": "c1e7571c"
-      },
-      "source": [
-        "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1F2ksmkoGQPa4pzRjMOE6BXWeOxWFIW6n?usp=sharing)\n",
-        "\n",
-        "# Llama Stack - Building AI Applications\n",
-        "\n",
-        "\"drawing\"\n",
-        "\n",
-        "[Llama Stack](https://github.com/meta-llama/llama-stack) defines and standardizes the set of core building blocks needed to bring generative AI applications to market. These building blocks are presented in the form of interoperable APIs with a broad set of Service Providers providing their implementations.\n",
-        "\n",
-        "Read more about the project: https://llama-stack.readthedocs.io/en/latest/index.html\n",
-        "\n",
-        "In this guide, we will showcase how you can build LLM-powered agentic applications using Llama Stack.\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "id": "4CV1Q19BDMVw",
-      "metadata": {
-        "id": "4CV1Q19BDMVw"
-      },
-      "source": [
-        "## 1. Getting started with Llama Stack"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "id": "K4AvfUAJZOeS",
-      "metadata": {
-        "id": "K4AvfUAJZOeS"
-      },
-      "source": [
-        "### 1.1. Create TogetherAI account\n",
-        "\n",
-        "\n",
-        "In order to run inference for the llama models, you will need to use an inference provider. Llama stack supports a number of inference [providers](https://github.com/meta-llama/llama-stack/tree/main/llama_stack/providers/remote/inference).\n",
-        "\n",
-        "\n",
-        "In this showcase, we will use [together.ai](https://www.together.ai/) as the inference provider. So, you would first get an API key from Together if you dont have one already.\n",
-        "\n",
-        "Steps [here](https://docs.google.com/document/d/1Vg998IjRW_uujAPnHdQ9jQWvtmkZFt74FldW2MblxPY/edit?usp=sharing).\n",
-        "\n",
-        "You can also use Fireworks.ai or even Ollama if you would like to.\n",
-        "\n",
-        "\n",
-        "\n",
-        "> **Note:**  Set the API Key in the Secrets of this notebook\n",
-        "\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "id": "oDUB7M_qe-Gs",
-      "metadata": {
-        "id": "oDUB7M_qe-Gs"
-      },
-      "source": [
-        "### 1.2. Install Llama Stack\n",
-        "\n",
-        "We will now start with installing the [llama-stack pypi package](https://pypi.org/project/llama-stack).\n",
-        "\n",
-        "In addition, we will install [bubblewrap](https://github.com/containers/bubblewrap), a low level light-weight container framework that runs in the user namespace. We will use it to execute code generated by Llama in one of the examples."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 1,
-      "id": "J2kGed0R5PSf",
-      "metadata": {
-        "colab": {
-          "base_uri": "https://localhost:8080/"
-        },
-        "collapsed": true,
-        "id": "J2kGed0R5PSf",
-        "outputId": "3fa6d087-2f12-444f-b3d3-9331305abb51"
-      },
-      "outputs": [
-        {
-          "output_type": "stream",
-          "name": "stdout",
-          "text": [
-            "Reading package lists... Done\n",
-            "Building dependency tree... Done\n",
-            "Reading state information... Done\n",
-            "The following NEW packages will be installed:\n",
-            "  bubblewrap\n",
-            "0 upgraded, 1 newly installed, 0 to remove and 49 not upgraded.\n",
-            "Need to get 46.3 kB of archives.\n",
-            "After this operation, 132 kB of additional disk space will be used.\n",
-            "Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 bubblewrap amd64 0.6.1-1ubuntu0.1 [46.3 kB]\n",
-            "Fetched 46.3 kB in 1s (52.2 kB/s)\n",
-            "Selecting previously unselected package bubblewrap.\n",
-            "(Reading database ... 123632 files and directories currently installed.)\n",
-            "Preparing to unpack .../bubblewrap_0.6.1-1ubuntu0.1_amd64.deb ...\n",
-            "Unpacking bubblewrap (0.6.1-1ubuntu0.1) ...\n",
-            "Setting up bubblewrap (0.6.1-1ubuntu0.1) ...\n",
-            "Processing triggers for man-db (2.10.2-1) ...\n",
-            "Collecting llama-stack-client@ git+https://github.com/meta-llama/llama-stack-client-python.git\n",
-            "  Cloning https://github.com/meta-llama/llama-stack-client-python.git to /tmp/pip-install-y4g346dn/llama-stack-client_dea5c21edaf144f4b76e5cb6f78c1a79\n",
-            "  Running command git clone --filter=blob:none --quiet https://github.com/meta-llama/llama-stack-client-python.git /tmp/pip-install-y4g346dn/llama-stack-client_dea5c21edaf144f4b76e5cb6f78c1a79\n",
-            "  Resolved https://github.com/meta-llama/llama-stack-client-python.git to commit db90c54d82e3c2fa6f334adcaf700940dad163a3\n",
-            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
-            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
-            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
-            "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-stack-client@ git+https://github.com/meta-llama/llama-stack-client-python.git) (3.7.1)\n",
-            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from llama-stack-client@ git+https://github.com/meta-llama/llama-stack-client-python.git) (8.1.8)\n",
-            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/local/lib/python3.10/dist-packages (from llama-stack-client@ git+https://github.com/meta-llama/llama-stack-client-python.git) (1.9.0)\n",
-            "Requirement already satisfied: httpx<1,>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from llama-stack-client@ git+https://github.com/meta-llama/llama-stack-client-python.git) (0.28.1)\n",
-            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from llama-stack-client@ git+https://github.com/meta-llama/llama-stack-client-python.git) (2.2.2)\n",
-            "Requirement already satisfied: prompt-toolkit in /usr/local/lib/python3.10/dist-packages (from llama-stack-client@ git+https://github.com/meta-llama/llama-stack-client-python.git) (3.0.48)\n",
-            "Collecting pyaml (from llama-stack-client@ git+https://github.com/meta-llama/llama-stack-client-python.git)\n",
-            "  Downloading pyaml-25.1.0-py3-none-any.whl.metadata (12 kB)\n",
-            "Requirement already satisfied: pydantic<3,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from llama-stack-client@ git+https://github.com/meta-llama/llama-stack-client-python.git) (2.10.4)\n",
-            "Requirement already satisfied: rich in /usr/local/lib/python3.10/dist-packages (from llama-stack-client@ git+https://github.com/meta-llama/llama-stack-client-python.git) (13.9.4)\n",
-            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from llama-stack-client@ git+https://github.com/meta-llama/llama-stack-client-python.git) (1.3.1)\n",
-            "Requirement already satisfied: termcolor in /usr/local/lib/python3.10/dist-packages (from llama-stack-client@ git+https://github.com/meta-llama/llama-stack-client-python.git) (2.5.0)\n",
-            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from llama-stack-client@ git+https://github.com/meta-llama/llama-stack-client-python.git) (4.67.1)\n",
-            "Requirement already satisfied: typing-extensions<5,>=4.7 in /usr/local/lib/python3.10/dist-packages (from llama-stack-client@ git+https://github.com/meta-llama/llama-stack-client-python.git) (4.12.2)\n",
-            "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->llama-stack-client@ git+https://github.com/meta-llama/llama-stack-client-python.git) (3.10)\n",
-            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->llama-stack-client@ git+https://github.com/meta-llama/llama-stack-client-python.git) (1.2.2)\n",
-            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->llama-stack-client@ git+https://github.com/meta-llama/llama-stack-client-python.git) (2024.12.14)\n",
-            "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->llama-stack-client@ git+https://github.com/meta-llama/llama-stack-client-python.git) (1.0.7)\n",
-            "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx<1,>=0.23.0->llama-stack-client@ git+https://github.com/meta-llama/llama-stack-client-python.git) (0.14.0)\n",
-            "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->llama-stack-client@ git+https://github.com/meta-llama/llama-stack-client-python.git) (0.7.0)\n",
-            "Requirement already satisfied: pydantic-core==2.27.2 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->llama-stack-client@ git+https://github.com/meta-llama/llama-stack-client-python.git) (2.27.2)\n",
-            "Requirement already satisfied: numpy>=1.22.4 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-stack-client@ git+https://github.com/meta-llama/llama-stack-client-python.git) (1.26.4)\n",
-            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-stack-client@ git+https://github.com/meta-llama/llama-stack-client-python.git) (2.8.2)\n",
-            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-stack-client@ git+https://github.com/meta-llama/llama-stack-client-python.git) (2024.2)\n",
-            "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-stack-client@ git+https://github.com/meta-llama/llama-stack-client-python.git) (2024.2)\n",
-            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.10/dist-packages (from prompt-toolkit->llama-stack-client@ git+https://github.com/meta-llama/llama-stack-client-python.git) (0.2.13)\n",
-            "Requirement already satisfied: PyYAML in /usr/local/lib/python3.10/dist-packages (from pyaml->llama-stack-client@ git+https://github.com/meta-llama/llama-stack-client-python.git) (6.0.2)\n",
-            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich->llama-stack-client@ git+https://github.com/meta-llama/llama-stack-client-python.git) (3.0.0)\n",
-            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich->llama-stack-client@ git+https://github.com/meta-llama/llama-stack-client-python.git) (2.18.0)\n",
-            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich->llama-stack-client@ git+https://github.com/meta-llama/llama-stack-client-python.git) (0.1.2)\n",
-            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas->llama-stack-client@ git+https://github.com/meta-llama/llama-stack-client-python.git) (1.17.0)\n",
-            "Downloading pyaml-25.1.0-py3-none-any.whl (26 kB)\n",
-            "Building wheels for collected packages: llama-stack-client\n",
-            "  Building wheel for llama-stack-client (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
-            "  Created wheel for llama-stack-client: filename=llama_stack_client-0.0.63-py3-none-any.whl size=318443 sha256=212ae3a9f3d5bb8a88801e4c3e625d99c9cb1d50d978cb6b2a8f7d069f013f06\n",
-            "  Stored in directory: /tmp/pip-ephem-wheel-cache-c7a22578/wheels/c9/21/63/5f6965968ab3dae8a0b1a0e43ca4991732ca03184aa158c15c\n",
-            "Successfully built llama-stack-client\n",
-            "Installing collected packages: pyaml, llama-stack-client\n",
-            "Successfully installed llama-stack-client-0.0.63 pyaml-25.1.0\n",
-            "Collecting llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor\n",
-            "  Cloning https://github.com/meta-llama/llama-stack.git (to revision fix_sqlite_span_processor) to /tmp/pip-install-0iqgax6t/llama-stack_824f45a9298043deacb6c11e12206393\n",
-            "  Running command git clone --filter=blob:none --quiet https://github.com/meta-llama/llama-stack.git /tmp/pip-install-0iqgax6t/llama-stack_824f45a9298043deacb6c11e12206393\n",
-            "  Running command git checkout -b fix_sqlite_span_processor --track origin/fix_sqlite_span_processor\n",
-            "  Switched to a new branch 'fix_sqlite_span_processor'\n",
-            "  Branch 'fix_sqlite_span_processor' set up to track remote branch 'fix_sqlite_span_processor' from 'origin'.\n",
-            "  Resolved https://github.com/meta-llama/llama-stack.git to commit 6fc155f25261691613d075fd8d08f728c2596815\n",
-            "  Running command git submodule update --init --recursive -q\n",
-            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
-            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
-            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
-            "Collecting blobfile (from llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor)\n",
-            "  Downloading blobfile-3.0.0-py3-none-any.whl.metadata (15 kB)\n",
-            "Collecting fire (from llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor)\n",
-            "  Downloading fire-0.7.0.tar.gz (87 kB)\n",
-            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m87.2/87.2 kB\u001b[0m \u001b[31m8.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
-            "Requirement already satisfied: httpx in /usr/local/lib/python3.10/dist-packages (from llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (0.28.1)\n",
-            "Requirement already satisfied: huggingface-hub in /usr/local/lib/python3.10/dist-packages (from llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (0.27.1)\n",
-            "Collecting llama-models>=0.0.63 (from llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor)\n",
-            "  Downloading llama_models-0.0.63-py3-none-any.whl.metadata (8.2 kB)\n",
-            "Requirement already satisfied: llama-stack-client>=0.0.63 in /usr/local/lib/python3.10/dist-packages (from llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (0.0.63)\n",
-            "Requirement already satisfied: prompt-toolkit in /usr/local/lib/python3.10/dist-packages (from llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (3.0.48)\n",
-            "Collecting python-dotenv (from llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor)\n",
-            "  Downloading python_dotenv-1.0.1-py3-none-any.whl.metadata (23 kB)\n",
-            "Requirement already satisfied: pydantic>=2 in /usr/local/lib/python3.10/dist-packages (from llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (2.10.4)\n",
-            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (2.32.3)\n",
-            "Requirement already satisfied: rich in /usr/local/lib/python3.10/dist-packages (from llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (13.9.4)\n",
-            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (75.1.0)\n",
-            "Requirement already satisfied: termcolor in /usr/local/lib/python3.10/dist-packages (from llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (2.5.0)\n",
-            "Requirement already satisfied: PyYAML in /usr/local/lib/python3.10/dist-packages (from llama-models>=0.0.63->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (6.0.2)\n",
-            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from llama-models>=0.0.63->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (3.1.5)\n",
-            "Collecting tiktoken (from llama-models>=0.0.63->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor)\n",
-            "  Downloading tiktoken-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.6 kB)\n",
-            "Requirement already satisfied: Pillow in /usr/local/lib/python3.10/dist-packages (from llama-models>=0.0.63->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (11.1.0)\n",
-            "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.63->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (3.7.1)\n",
-            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.63->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (8.1.8)\n",
-            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.63->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (1.9.0)\n",
-            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.63->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (2.2.2)\n",
-            "Requirement already satisfied: pyaml in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.63->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (25.1.0)\n",
-            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.63->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (1.3.1)\n",
-            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.63->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (4.67.1)\n",
-            "Requirement already satisfied: typing-extensions<5,>=4.7 in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.63->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (4.12.2)\n",
-            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (2024.12.14)\n",
-            "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (1.0.7)\n",
-            "Requirement already satisfied: idna in /usr/local/lib/python3.10/dist-packages (from httpx->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (3.10)\n",
-            "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (0.14.0)\n",
-            "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.10/dist-packages (from pydantic>=2->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (0.7.0)\n",
-            "Requirement already satisfied: pydantic-core==2.27.2 in /usr/local/lib/python3.10/dist-packages (from pydantic>=2->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (2.27.2)\n",
-            "Collecting pycryptodomex>=3.8 (from blobfile->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor)\n",
-            "  Downloading pycryptodomex-3.21.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.4 kB)\n",
-            "Requirement already satisfied: urllib3<3,>=1.25.3 in /usr/local/lib/python3.10/dist-packages (from blobfile->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (2.3.0)\n",
-            "Requirement already satisfied: lxml>=4.9 in /usr/local/lib/python3.10/dist-packages (from blobfile->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (5.3.0)\n",
-            "Requirement already satisfied: filelock>=3.0 in /usr/local/lib/python3.10/dist-packages (from blobfile->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (3.16.1)\n",
-            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (2024.10.0)\n",
-            "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (24.2)\n",
-            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.10/dist-packages (from prompt-toolkit->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (0.2.13)\n",
-            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (3.4.1)\n",
-            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (3.0.0)\n",
-            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (2.18.0)\n",
-            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->llama-stack-client>=0.0.63->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (1.2.2)\n",
-            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (0.1.2)\n",
-            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->llama-models>=0.0.63->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (3.0.2)\n",
-            "Requirement already satisfied: numpy>=1.22.4 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-stack-client>=0.0.63->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (1.26.4)\n",
-            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-stack-client>=0.0.63->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (2.8.2)\n",
-            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-stack-client>=0.0.63->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (2024.2)\n",
-            "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-stack-client>=0.0.63->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (2024.2)\n",
-            "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken->llama-models>=0.0.63->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (2024.11.6)\n",
-            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas->llama-stack-client>=0.0.63->llama-stack@ git+https://github.com/meta-llama/llama-stack.git@fix_sqlite_span_processor) (1.17.0)\n",
-            "Downloading llama_models-0.0.63-py3-none-any.whl (1.6 MB)\n",
-            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m48.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-            "\u001b[?25hDownloading blobfile-3.0.0-py3-none-any.whl (75 kB)\n",
-            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.4/75.4 kB\u001b[0m \u001b[31m7.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-            "\u001b[?25hDownloading python_dotenv-1.0.1-py3-none-any.whl (19 kB)\n",
-            "Downloading pycryptodomex-3.21.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.3 MB)\n",
-            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.3/2.3 MB\u001b[0m \u001b[31m67.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-            "\u001b[?25hDownloading tiktoken-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
-            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m60.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-            "\u001b[?25hBuilding wheels for collected packages: llama-stack, fire\n",
-            "  Building wheel for llama-stack (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
-            "  Created wheel for llama-stack: filename=llama_stack-0.0.63-py3-none-any.whl size=500660 sha256=36cd6d1b0146d456976f2d64deddf31a6515e5b0fbee97b61e448eb10356f3a7\n",
-            "  Stored in directory: /tmp/pip-ephem-wheel-cache-qw3m4ho9/wheels/47/17/a3/49a8b1238e1c4640a5fdce6ad5055df118b069a670e77876e2\n",
-            "  Building wheel for fire (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
-            "  Created wheel for fire: filename=fire-0.7.0-py3-none-any.whl size=114249 sha256=c1175a999f843dbb0dcabbeae06a6b080f59d7f78171dd089824c37fd63aeaef\n",
-            "  Stored in directory: /root/.cache/pip/wheels/19/39/2f/2d3cadc408a8804103f1c34ddd4b9f6a93497b11fa96fe738e\n",
-            "Successfully built llama-stack fire\n",
-            "Installing collected packages: python-dotenv, pycryptodomex, fire, tiktoken, blobfile, llama-models, llama-stack\n",
-            "Successfully installed blobfile-3.0.0 fire-0.7.0 llama-models-0.0.63 llama-stack-0.0.63 pycryptodomex-3.21.0 python-dotenv-1.0.1 tiktoken-0.8.0\n"
-          ]
-        }
-      ],
-      "source": [
-        "!apt-get install -y bubblewrap\n",
-        "# install a branch of llama stack\n",
-        "!pip install llama-stack"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "id": "414301dc",
-      "metadata": {
-        "id": "414301dc"
-      },
-      "source": [
-        "### 1.3. Configure Llama Stack for Together\n",
-        "\n",
-        "\n",
-        "Llama Stack is architected as a collection of lego blocks which can be assembled as needed.\n",
-        "\n",
-        "\n",
-        "Typically, llama stack is available as a server with an endpoint that you can hit. We call this endpoint a [Distribution](https://llama-stack.readthedocs.io/en/latest/concepts/index.html#distributions). Partners like Together and Fireworks offer their own Llama Stack Distribution endpoints.\n",
-        "\n",
-        "In this showcase, we are going to use llama stack inline as a library. So, given a particular set of providers, we must first package up the right set of dependencies. We have a template to use Together as an inference provider and [faiss](https://ai.meta.com/tools/faiss/) for memory/RAG.\n",
-        "\n",
-        "We will run `llama stack build` to deploy all dependencies."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 2,
-      "id": "HaepEZXCDgif",
-      "metadata": {
-        "colab": {
-          "base_uri": "https://localhost:8080/"
-        },
-        "collapsed": true,
-        "id": "HaepEZXCDgif",
-        "outputId": "6c983bb7-1cbe-4249-fd0a-0c629851981b"
-      },
-      "outputs": [
-        {
-          "output_type": "stream",
-          "name": "stdout",
-          "text": [
-            "Requirement already satisfied: llama-stack in /usr/local/lib/python3.10/dist-packages (0.0.63)\r\n",
-            "Requirement already satisfied: blobfile in /usr/local/lib/python3.10/dist-packages (from llama-stack) (3.0.0)\r\n",
-            "Requirement already satisfied: fire in /usr/local/lib/python3.10/dist-packages (from llama-stack) (0.7.0)\r\n",
-            "Requirement already satisfied: httpx in /usr/local/lib/python3.10/dist-packages (from llama-stack) (0.28.1)\r\n",
-            "Requirement already satisfied: huggingface-hub in /usr/local/lib/python3.10/dist-packages (from llama-stack) (0.27.1)\r\n",
-            "Requirement already satisfied: llama-models>=0.0.63 in /usr/local/lib/python3.10/dist-packages (from llama-stack) (0.0.63)\r\n",
-            "Requirement already satisfied: llama-stack-client>=0.0.63 in /usr/local/lib/python3.10/dist-packages (from llama-stack) (0.0.63)\r\n",
-            "Requirement already satisfied: prompt-toolkit in /usr/local/lib/python3.10/dist-packages (from llama-stack) (3.0.48)\r\n",
-            "Requirement already satisfied: python-dotenv in /usr/local/lib/python3.10/dist-packages (from llama-stack) (1.0.1)\r\n",
-            "Requirement already satisfied: pydantic>=2 in /usr/local/lib/python3.10/dist-packages (from llama-stack) (2.10.4)\r\n",
-            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from llama-stack) (2.32.3)\r\n",
-            "Requirement already satisfied: rich in /usr/local/lib/python3.10/dist-packages (from llama-stack) (13.9.4)\r\n",
-            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from llama-stack) (75.1.0)\r\n",
-            "Requirement already satisfied: termcolor in /usr/local/lib/python3.10/dist-packages (from llama-stack) (2.5.0)\r\n",
-            "Requirement already satisfied: PyYAML in /usr/local/lib/python3.10/dist-packages (from llama-models>=0.0.63->llama-stack) (6.0.2)\r\n",
-            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from llama-models>=0.0.63->llama-stack) (3.1.5)\r\n",
-            "Requirement already satisfied: tiktoken in /usr/local/lib/python3.10/dist-packages (from llama-models>=0.0.63->llama-stack) (0.8.0)\r\n",
-            "Requirement already satisfied: Pillow in /usr/local/lib/python3.10/dist-packages (from llama-models>=0.0.63->llama-stack) (11.1.0)\r\n",
-            "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.63->llama-stack) (3.7.1)\r\n",
-            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.63->llama-stack) (8.1.8)\r\n",
-            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.63->llama-stack) (1.9.0)\r\n",
-            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.63->llama-stack) (2.2.2)\r\n",
-            "Requirement already satisfied: pyaml in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.63->llama-stack) (25.1.0)\r\n",
-            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.63->llama-stack) (1.3.1)\r\n",
-            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.63->llama-stack) (4.67.1)\r\n",
-            "Requirement already satisfied: typing-extensions<5,>=4.7 in /usr/local/lib/python3.10/dist-packages (from llama-stack-client>=0.0.63->llama-stack) (4.12.2)\r\n",
-            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx->llama-stack) (2024.12.14)\r\n",
-            "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx->llama-stack) (1.0.7)\r\n",
-            "Requirement already satisfied: idna in /usr/local/lib/python3.10/dist-packages (from httpx->llama-stack) (3.10)\r\n",
-            "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx->llama-stack) (0.14.0)\r\n",
-            "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.10/dist-packages (from pydantic>=2->llama-stack) (0.7.0)\r\n",
-            "Requirement already satisfied: pydantic-core==2.27.2 in /usr/local/lib/python3.10/dist-packages (from pydantic>=2->llama-stack) (2.27.2)\r\n",
-            "Requirement already satisfied: pycryptodomex>=3.8 in /usr/local/lib/python3.10/dist-packages (from blobfile->llama-stack) (3.21.0)\r\n",
-            "Requirement already satisfied: urllib3<3,>=1.25.3 in /usr/local/lib/python3.10/dist-packages (from blobfile->llama-stack) (2.3.0)\r\n",
-            "Requirement already satisfied: lxml>=4.9 in /usr/local/lib/python3.10/dist-packages (from blobfile->llama-stack) (5.3.0)\r\n",
-            "Requirement already satisfied: filelock>=3.0 in /usr/local/lib/python3.10/dist-packages (from blobfile->llama-stack) (3.16.1)\r\n",
-            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->llama-stack) (2024.10.0)\r\n",
-            "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->llama-stack) (24.2)\r\n",
-            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.10/dist-packages (from prompt-toolkit->llama-stack) (0.2.13)\r\n",
-            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->llama-stack) (3.4.1)\r\n",
-            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich->llama-stack) (3.0.0)\r\n",
-            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich->llama-stack) (2.18.0)\r\n",
-            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->llama-stack-client>=0.0.63->llama-stack) (1.2.2)\r\n",
-            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich->llama-stack) (0.1.2)\r\n",
-            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->llama-models>=0.0.63->llama-stack) (3.0.2)\n",
-            "Requirement already satisfied: numpy>=1.22.4 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-stack-client>=0.0.63->llama-stack) (1.26.4)\n",
-            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-stack-client>=0.0.63->llama-stack) (2.8.2)\n",
-            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-stack-client>=0.0.63->llama-stack) (2024.2)\n",
-            "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas->llama-stack-client>=0.0.63->llama-stack) (2024.2)\n",
-            "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken->llama-models>=0.0.63->llama-stack) (2024.11.6)\n",
-            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas->llama-stack-client>=0.0.63->llama-stack) (1.17.0)\n",
-            "Installing pip dependencies\n",
-            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (1.6.0)\n",
-            "Collecting psycopg2-binary\n",
-            "  Downloading psycopg2_binary-2.9.10-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.9 kB)\n",
-            "Collecting autoevals\n",
-            "  Downloading autoevals-0.0.115-py3-none-any.whl.metadata (12 kB)\n",
-            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (1.13.1)\n",
-            "Collecting pypdf\n",
-            "  Downloading pypdf-5.1.0-py3-none-any.whl.metadata (7.2 kB)\n",
-            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (2.2.2)\n",
-            "Collecting datasets\n",
-            "  Downloading datasets-3.2.0-py3-none-any.whl.metadata (20 kB)\n",
-            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (4.67.1)\n",
-            "Requirement already satisfied: opentelemetry-sdk in /usr/local/lib/python3.10/dist-packages (1.29.0)\n",
-            "Requirement already satisfied: openai in /usr/local/lib/python3.10/dist-packages (1.59.4)\n",
-            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (2.32.3)\n",
-            "Collecting opentelemetry-exporter-otlp-proto-http\n",
-            "  Downloading opentelemetry_exporter_otlp_proto_http-1.29.0-py3-none-any.whl.metadata (2.2 kB)\n",
-            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (1.26.4)\n",
-            "Collecting together\n",
-            "  Downloading together-1.3.11-py3-none-any.whl.metadata (11 kB)\n",
-            "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.47.1)\n",
-            "Requirement already satisfied: chardet in /usr/local/lib/python3.10/dist-packages (5.2.0)\n",
-            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (3.10.0)\n",
-            "Requirement already satisfied: pillow in /usr/local/lib/python3.10/dist-packages (11.1.0)\n",
-            "Collecting faiss-cpu\n",
-            "  Downloading faiss_cpu-1.9.0.post1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.4 kB)\n",
-            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.10/dist-packages (0.2.0)\n",
-            "Collecting redis\n",
-            "  Downloading redis-5.2.1-py3-none-any.whl.metadata (9.1 kB)\n",
-            "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (3.9.1)\n",
-            "Collecting chromadb-client\n",
-            "  Downloading chromadb_client-0.6.2-py3-none-any.whl.metadata (2.4 kB)\n",
-            "Requirement already satisfied: blobfile in /usr/local/lib/python3.10/dist-packages (3.0.0)\n",
-            "Collecting aiosqlite\n",
-            "  Downloading aiosqlite-0.20.0-py3-none-any.whl.metadata (4.3 kB)\n",
-            "Collecting fastapi\n",
-            "  Downloading fastapi-0.115.6-py3-none-any.whl.metadata (27 kB)\n",
-            "Requirement already satisfied: fire in /usr/local/lib/python3.10/dist-packages (0.7.0)\n",
-            "Requirement already satisfied: httpx in /usr/local/lib/python3.10/dist-packages (0.28.1)\n",
-            "Collecting uvicorn\n",
-            "  Downloading uvicorn-0.34.0-py3-none-any.whl.metadata (6.5 kB)\n",
-            "Requirement already satisfied: joblib>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn) (1.4.2)\n",
-            "Requirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn) (3.5.0)\n",
-            "Collecting chevron (from autoevals)\n",
-            "  Downloading chevron-0.14.0-py3-none-any.whl.metadata (4.9 kB)\n",
-            "Collecting levenshtein (from autoevals)\n",
-            "  Downloading levenshtein-0.26.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.2 kB)\n",
-            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from autoevals) (6.0.2)\n",
-            "Collecting braintrust_core==0.0.57 (from autoevals)\n",
-            "  Downloading braintrust_core-0.0.57-py3-none-any.whl.metadata (669 bytes)\n",
-            "Requirement already satisfied: jsonschema in /usr/local/lib/python3.10/dist-packages (from autoevals) (4.23.0)\n",
-            "Requirement already satisfied: typing_extensions>=4.0 in /usr/local/lib/python3.10/dist-packages (from pypdf) (4.12.2)\n",
-            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas) (2.8.2)\n",
-            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas) (2024.2)\n",
-            "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas) (2024.2)\n",
-            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from datasets) (3.16.1)\n",
-            "Requirement already satisfied: pyarrow>=15.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (17.0.0)\n",
-            "Collecting dill<0.3.9,>=0.3.0 (from datasets)\n",
-            "  Downloading dill-0.3.8-py3-none-any.whl.metadata (10 kB)\n",
-            "Collecting xxhash (from datasets)\n",
-            "  Downloading xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (12 kB)\n",
-            "Collecting multiprocess<0.70.17 (from datasets)\n",
-            "  Downloading multiprocess-0.70.16-py310-none-any.whl.metadata (7.2 kB)\n",
-            "Collecting fsspec<=2024.9.0,>=2023.1.0 (from fsspec[http]<=2024.9.0,>=2023.1.0->datasets)\n",
-            "  Downloading fsspec-2024.9.0-py3-none-any.whl.metadata (11 kB)\n",
-            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.11.11)\n",
-            "Requirement already satisfied: huggingface-hub>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.27.1)\n",
-            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from datasets) (24.2)\n",
-            "Requirement already satisfied: opentelemetry-api==1.29.0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-sdk) (1.29.0)\n",
-            "Requirement already satisfied: opentelemetry-semantic-conventions==0.50b0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-sdk) (0.50b0)\n",
-            "Requirement already satisfied: deprecated>=1.2.6 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-api==1.29.0->opentelemetry-sdk) (1.2.15)\n",
-            "Requirement already satisfied: importlib-metadata<=8.5.0,>=6.0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-api==1.29.0->opentelemetry-sdk) (8.5.0)\n",
-            "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai) (3.7.1)\n",
-            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/local/lib/python3.10/dist-packages (from openai) (1.9.0)\n",
-            "Requirement already satisfied: jiter<1,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from openai) (0.8.2)\n",
-            "Requirement already satisfied: pydantic<3,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from openai) (2.10.4)\n",
-            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai) (1.3.1)\n",
-            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests) (3.4.1)\n",
-            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests) (3.10)\n",
-            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests) (2.3.0)\n",
-            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests) (2024.12.14)\n",
-            "Requirement already satisfied: googleapis-common-protos~=1.52 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-exporter-otlp-proto-http) (1.66.0)\n",
-            "Collecting opentelemetry-exporter-otlp-proto-common==1.29.0 (from opentelemetry-exporter-otlp-proto-http)\n",
-            "  Downloading opentelemetry_exporter_otlp_proto_common-1.29.0-py3-none-any.whl.metadata (1.8 kB)\n",
-            "Collecting opentelemetry-proto==1.29.0 (from opentelemetry-exporter-otlp-proto-http)\n",
-            "  Downloading opentelemetry_proto-1.29.0-py3-none-any.whl.metadata (2.3 kB)\n",
-            "Collecting protobuf<6.0,>=5.0 (from opentelemetry-proto==1.29.0->opentelemetry-exporter-otlp-proto-http)\n",
-            "  Downloading protobuf-5.29.3-cp38-abi3-manylinux2014_x86_64.whl.metadata (592 bytes)\n",
-            "Requirement already satisfied: click<9.0.0,>=8.1.7 in /usr/local/lib/python3.10/dist-packages (from together) (8.1.8)\n",
-            "Requirement already satisfied: eval-type-backport<0.3.0,>=0.1.3 in /usr/local/lib/python3.10/dist-packages (from together) (0.2.2)\n",
-            "Collecting pillow\n",
-            "  Downloading pillow-10.4.0-cp310-cp310-manylinux_2_28_x86_64.whl.metadata (9.2 kB)\n",
-            "Requirement already satisfied: rich<14.0.0,>=13.8.1 in /usr/local/lib/python3.10/dist-packages (from together) (13.9.4)\n",
-            "Requirement already satisfied: tabulate<0.10.0,>=0.9.0 in /usr/local/lib/python3.10/dist-packages (from together) (0.9.0)\n",
-            "Requirement already satisfied: typer<0.16,>=0.9 in /usr/local/lib/python3.10/dist-packages (from together) (0.15.1)\n",
-            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2024.11.6)\n",
-            "Requirement already satisfied: tokenizers<0.22,>=0.21 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.21.0)\n",
-            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.5.1)\n",
-            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (1.3.1)\n",
-            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (0.12.1)\n",
-            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (4.55.3)\n",
-            "Requirement already satisfied: kiwisolver>=1.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (1.4.8)\n",
-            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (3.2.1)\n",
-            "Requirement already satisfied: async-timeout>=4.0.3 in /usr/local/lib/python3.10/dist-packages (from redis) (4.0.3)\n",
-            "Collecting opentelemetry-exporter-otlp-proto-grpc>=1.2.0 (from chromadb-client)\n",
-            "  Downloading opentelemetry_exporter_otlp_proto_grpc-1.29.0-py3-none-any.whl.metadata (2.2 kB)\n",
-            "Collecting overrides>=7.3.1 (from chromadb-client)\n",
-            "  Downloading overrides-7.7.0-py3-none-any.whl.metadata (5.8 kB)\n",
-            "Collecting posthog>=2.4.0 (from chromadb-client)\n",
-            "  Downloading posthog-3.7.5-py2.py3-none-any.whl.metadata (2.0 kB)\n",
-            "Requirement already satisfied: tenacity>=8.2.3 in /usr/local/lib/python3.10/dist-packages (from chromadb-client) (9.0.0)\n",
-            "Requirement already satisfied: orjson>=3.9.12 in /usr/local/lib/python3.10/dist-packages (from chromadb-client) (3.10.13)\n",
-            "Requirement already satisfied: pycryptodomex>=3.8 in /usr/local/lib/python3.10/dist-packages (from blobfile) (3.21.0)\n",
-            "Requirement already satisfied: lxml>=4.9 in /usr/local/lib/python3.10/dist-packages (from blobfile) (5.3.0)\n",
-            "Collecting starlette<0.42.0,>=0.40.0 (from fastapi)\n",
-            "  Downloading starlette-0.41.3-py3-none-any.whl.metadata (6.0 kB)\n",
-            "Requirement already satisfied: termcolor in /usr/local/lib/python3.10/dist-packages (from fire) (2.5.0)\n",
-            "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx) (1.0.7)\n",
-            "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx) (0.14.0)\n",
-            "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (2.4.4)\n",
-            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.2)\n",
-            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (24.3.0)\n",
-            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.5.0)\n",
-            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.1.0)\n",
-            "Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (0.2.1)\n",
-            "Requirement already satisfied: yarl<2.0,>=1.17.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.18.3)\n",
-            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai) (1.2.2)\n",
-            "Requirement already satisfied: wrapt<2,>=1.10 in /usr/local/lib/python3.10/dist-packages (from deprecated>=1.2.6->opentelemetry-api==1.29.0->opentelemetry-sdk) (1.17.0)\n",
-            "Requirement already satisfied: grpcio<2.0.0,>=1.63.2 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb-client) (1.69.0)\n",
-            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from posthog>=2.4.0->chromadb-client) (1.17.0)\n",
-            "Collecting monotonic>=1.5 (from posthog>=2.4.0->chromadb-client)\n",
-            "  Downloading monotonic-1.6-py2.py3-none-any.whl.metadata (1.5 kB)\n",
-            "Collecting backoff>=1.10.0 (from posthog>=2.4.0->chromadb-client)\n",
-            "  Downloading backoff-2.2.1-py3-none-any.whl.metadata (14 kB)\n",
-            "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai) (0.7.0)\n",
-            "Requirement already satisfied: pydantic-core==2.27.2 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai) (2.27.2)\n",
-            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich<14.0.0,>=13.8.1->together) (3.0.0)\n",
-            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich<14.0.0,>=13.8.1->together) (2.18.0)\n",
-            "Requirement already satisfied: shellingham>=1.3.0 in /usr/local/lib/python3.10/dist-packages (from typer<0.16,>=0.9->together) (1.5.4)\n",
-            "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.10/dist-packages (from jsonschema->autoevals) (2024.10.1)\n",
-            "Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.10/dist-packages (from jsonschema->autoevals) (0.35.1)\n",
-            "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from jsonschema->autoevals) (0.22.3)\n",
-            "Collecting rapidfuzz<4.0.0,>=3.9.0 (from levenshtein->autoevals)\n",
-            "  Downloading rapidfuzz-3.11.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (11 kB)\n",
-            "Requirement already satisfied: zipp>=3.20 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata<=8.5.0,>=6.0->opentelemetry-api==1.29.0->opentelemetry-sdk) (3.21.0)\n",
-            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich<14.0.0,>=13.8.1->together) (0.1.2)\n",
-            "Downloading psycopg2_binary-2.9.10-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.0 MB)\n",
-            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.0/3.0 MB\u001b[0m \u001b[31m84.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-            "\u001b[?25hDownloading autoevals-0.0.115-py3-none-any.whl (41 kB)\n",
-            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.1/41.1 kB\u001b[0m \u001b[31m3.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-            "\u001b[?25hDownloading braintrust_core-0.0.57-py3-none-any.whl (4.4 kB)\n",
-            "Downloading pypdf-5.1.0-py3-none-any.whl (297 kB)\n",
-            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m298.0/298.0 kB\u001b[0m \u001b[31m29.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-            "\u001b[?25hDownloading datasets-3.2.0-py3-none-any.whl (480 kB)\n",
-            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m480.6/480.6 kB\u001b[0m \u001b[31m37.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-            "\u001b[?25hDownloading opentelemetry_exporter_otlp_proto_http-1.29.0-py3-none-any.whl (17 kB)\n",
-            "Downloading opentelemetry_exporter_otlp_proto_common-1.29.0-py3-none-any.whl (18 kB)\n",
-            "Downloading opentelemetry_proto-1.29.0-py3-none-any.whl (55 kB)\n",
-            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m55.8/55.8 kB\u001b[0m \u001b[31m5.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-            "\u001b[?25hDownloading together-1.3.11-py3-none-any.whl (70 kB)\n",
-            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m70.6/70.6 kB\u001b[0m \u001b[31m6.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-            "\u001b[?25hDownloading pillow-10.4.0-cp310-cp310-manylinux_2_28_x86_64.whl (4.5 MB)\n",
-            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.5/4.5 MB\u001b[0m \u001b[31m105.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-            "\u001b[?25hDownloading faiss_cpu-1.9.0.post1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (27.5 MB)\n",
-            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m27.5/27.5 MB\u001b[0m \u001b[31m78.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-            "\u001b[?25hDownloading redis-5.2.1-py3-none-any.whl (261 kB)\n",
-            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m261.5/261.5 kB\u001b[0m \u001b[31m23.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-            "\u001b[?25hDownloading chromadb_client-0.6.2-py3-none-any.whl (604 kB)\n",
-            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m604.2/604.2 kB\u001b[0m \u001b[31m47.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-            "\u001b[?25hDownloading aiosqlite-0.20.0-py3-none-any.whl (15 kB)\n",
-            "Downloading fastapi-0.115.6-py3-none-any.whl (94 kB)\n",
-            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m94.8/94.8 kB\u001b[0m \u001b[31m9.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-            "\u001b[?25hDownloading uvicorn-0.34.0-py3-none-any.whl (62 kB)\n",
-            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.3/62.3 kB\u001b[0m \u001b[31m5.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-            "\u001b[?25hDownloading dill-0.3.8-py3-none-any.whl (116 kB)\n",
-            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m9.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-            "\u001b[?25hDownloading fsspec-2024.9.0-py3-none-any.whl (179 kB)\n",
-            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m179.3/179.3 kB\u001b[0m \u001b[31m18.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-            "\u001b[?25hDownloading multiprocess-0.70.16-py310-none-any.whl (134 kB)\n",
-            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m14.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-            "\u001b[?25hDownloading opentelemetry_exporter_otlp_proto_grpc-1.29.0-py3-none-any.whl (18 kB)\n",
-            "Downloading overrides-7.7.0-py3-none-any.whl (17 kB)\n",
-            "Downloading posthog-3.7.5-py2.py3-none-any.whl (54 kB)\n",
-            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m54.9/54.9 kB\u001b[0m \u001b[31m5.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-            "\u001b[?25hDownloading starlette-0.41.3-py3-none-any.whl (73 kB)\n",
-            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m73.2/73.2 kB\u001b[0m \u001b[31m7.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-            "\u001b[?25hDownloading chevron-0.14.0-py3-none-any.whl (11 kB)\n",
-            "Downloading levenshtein-0.26.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (162 kB)\n",
-            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m162.6/162.6 kB\u001b[0m \u001b[31m16.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-            "\u001b[?25hDownloading xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n",
-            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m20.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-            "\u001b[?25hDownloading backoff-2.2.1-py3-none-any.whl (15 kB)\n",
-            "Downloading monotonic-1.6-py2.py3-none-any.whl (8.2 kB)\n",
-            "Downloading protobuf-5.29.3-cp38-abi3-manylinux2014_x86_64.whl (319 kB)\n",
-            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m319.7/319.7 kB\u001b[0m \u001b[31m26.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-            "\u001b[?25hDownloading rapidfuzz-3.11.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.1 MB)\n",
-            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m102.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
-            "\u001b[?25hInstalling collected packages: monotonic, chevron, xxhash, uvicorn, redis, rapidfuzz, pypdf, psycopg2-binary, protobuf, pillow, overrides, fsspec, faiss-cpu, dill, braintrust_core, backoff, aiosqlite, starlette, posthog, opentelemetry-proto, multiprocess, levenshtein, opentelemetry-exporter-otlp-proto-common, fastapi, together, autoevals, opentelemetry-exporter-otlp-proto-http, opentelemetry-exporter-otlp-proto-grpc, datasets, chromadb-client\n",
-            "  Attempting uninstall: protobuf\n",
-            "    Found existing installation: protobuf 4.25.5\n",
-            "    Uninstalling protobuf-4.25.5:\n",
-            "      Successfully uninstalled protobuf-4.25.5\n",
-            "  Attempting uninstall: pillow\n",
-            "    Found existing installation: pillow 11.1.0\n",
-            "    Uninstalling pillow-11.1.0:\n",
-            "      Successfully uninstalled pillow-11.1.0\n",
-            "  Attempting uninstall: fsspec\n",
-            "    Found existing installation: fsspec 2024.10.0\n",
-            "    Uninstalling fsspec-2024.10.0:\n",
-            "      Successfully uninstalled fsspec-2024.10.0\n",
-            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
-            "gcsfs 2024.10.0 requires fsspec==2024.10.0, but you have fsspec 2024.9.0 which is incompatible.\n",
-            "tensorflow 2.17.1 requires protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3, but you have protobuf 5.29.3 which is incompatible.\n",
-            "tensorflow-metadata 1.13.1 requires protobuf<5,>=3.20.3, but you have protobuf 5.29.3 which is incompatible.\u001b[0m\u001b[31m\n",
-            "\u001b[0mSuccessfully installed aiosqlite-0.20.0 autoevals-0.0.115 backoff-2.2.1 braintrust_core-0.0.57 chevron-0.14.0 chromadb-client-0.6.2 datasets-3.2.0 dill-0.3.8 faiss-cpu-1.9.0.post1 fastapi-0.115.6 fsspec-2024.9.0 levenshtein-0.26.1 monotonic-1.6 multiprocess-0.70.16 opentelemetry-exporter-otlp-proto-common-1.29.0 opentelemetry-exporter-otlp-proto-grpc-1.29.0 opentelemetry-exporter-otlp-proto-http-1.29.0 opentelemetry-proto-1.29.0 overrides-7.7.0 pillow-10.4.0 posthog-3.7.5 protobuf-5.29.3 psycopg2-binary-2.9.10 pypdf-5.1.0 rapidfuzz-3.11.0 redis-5.2.1 starlette-0.41.3 together-1.3.11 uvicorn-0.34.0 xxhash-3.5.0\n",
-            "sentence-transformers --no-deps\n",
-            "Requirement already satisfied: sentence-transformers in /usr/local/lib/python3.10/dist-packages (3.3.1)\n",
-            "torch --index-url https://download.pytorch.org/whl/cpu\n",
-            "Looking in indexes: https://download.pytorch.org/whl/cpu\n",
-            "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (2.5.1+cu121)\n",
-            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch) (3.16.1)\n",
-            "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch) (4.12.2)\n",
-            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch) (3.4.2)\n",
-            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch) (3.1.5)\n",
-            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch) (2024.9.0)\n",
-            "Requirement already satisfied: sympy==1.13.1 in /usr/local/lib/python3.10/dist-packages (from torch) (1.13.1)\n",
-            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy==1.13.1->torch) (1.3.0)\n",
-            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch) (3.0.2)\n",
-            "\u001b[32mBuild Successful!\u001b[0m\n"
-          ]
-        }
-      ],
-      "source": [
-        "# This will build all the dependencies you will need\n",
-        "!llama stack build --template together --image-type venv"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "id": "25b97dfe",
-      "metadata": {
-        "id": "25b97dfe"
-      },
-      "source": [
-        "### 1.4. Initialize Llama Stack\n",
-        "\n",
-        "Now that all dependencies have been installed, we can initialize llama stack. We will first set the `TOGETHER_API_KEY` environment variable\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": 3,
-      "id": "E1UFuJC570Tk",
-      "metadata": {
-        "colab": {
-          "base_uri": "https://localhost:8080/",
-          "height": 1000,
-          "referenced_widgets": [
-            "88f0c88612bb45d59f07e93567cc0e14",
-            "9b24a82117e1482a8f6665978e84089c",
-            "8e75bf7cac454eeabd5ce47a1e981c68",
-            "fc272883566541108f83117ccd146a21",
-            "2e27a025a416434f8ab3b63049626d11",
-            "3a46a46bc8124a92b27aef43cbc009b6",
-            "4ad6bc0cca62446d8faf19a341bfa86f",
-            "6437c99289f947449f7d2964288973e5",
-            "e2f7dea8fc744537b42d0f1a85a73eb4",
-            "1377d2160344430da8f29a50d113a288",
-            "0c0b30e126724f9282ac5acbcb4581db",
-            "895efd0b6d9f4b319159703d965d1966",
-            "dece6dff65394a5f93585c73359d4dad",
-            "1030c0848635497681cc9ff0c344fb1a",
-            "fa6ecaab432347de8427b9b5ac3d4524",
-            "5effefa8e3764e3aaff57fe0197a7c96",
-            "1756eceba2c34c1ca182b7db465e95ce",
-            "0fd62e56e0bb41a996c04e63381d2a29",
-            "29badfc2eb0345d38d7cfc6c7f8bb1a8",
-            "e64cedb4560a43d8a43f36002087ac30",
-            "45aadb26b382460eb5b6b147509fb75a",
-            "130f2f5840764e8dbd573cc8a6ea6f5f",
-            "9ee45247ec144bb3aafe4208f316063f",
-            "da330e0999cb4c3c91a1cb1026304568",
-            "ff58a5381fb74cb1b9efc10f5c2738d6",
-            "18ed62b1d4594ed9a2651fa5df046efc",
-            "4004cda1d84949f5a380536f8a9d0274",
-            "54bddcf41c5641b7a56c981aadb62ef1",
-            "a9a0d8415d9d4e98a3f02ae8ec1053da",
-            "cceff1126242494bab432205c7ac7345",
-            "e6e53c439dab4639adc1c3c873602476",
-            "95db8eab3f964edf99038ad53f41fabc",
-            "52f1d69c6cd04816b6f34657893ae32b",
-            "b79a1dfcf2904bcba332569dbf351f34",
-            "7363b1a9a1b54a57bf15357e897128fd",
-            "3ac596104cdc4439b3980f7ce66ad080",
-            "5c9ec25994914acd8e13866b3eb943e1",
-            "38a958036c6e4155815a8169f1be1e53",
-            "cf5113a647ce45c4a3a523361aa3b5af",
-            "da8c20a65ba541bda058614849d5cfe2",
-            "40e9f20d74374b0e82c653caa0559d04",
-            "f46cfc9237e64db6be2ec6529b61ec88",
-            "dc04575da46540d4ad3a708e58f0de6a",
-            "24c0be775e474517a7be49d187822bd0",
-            "111184729957441d9d1f3d404bd82757",
-            "be060f9d7a664c17a80510f447c0bee3",
-            "228445132e5f4b2ca793f4beeeca4426",
-            "b96a2e34a2af435b9705550fe564591d",
-            "1f1cdac013af4559889f15eebac5256a",
-            "834ae2d249b94be6bbe5349509536a4b",
-            "509863a58de74b07b813aa83ffa4a507",
-            "48a5b775a4324da791603b83d61be7d1",
-            "02b60dad91c7482ba70cf8bb954bc4eb",
-            "2bfb0fb5506d4285918a9c94af9ab5d1",
-            "0f699b0f99484a8ba2eb17bb1d621c5a",
-            "c6f34317390e4f90b16235f2ae84a981",
-            "3da95c8814f34472a181ce7687f9e15e",
-            "4d1c2de4c1354ef0b84c54c447141707",
-            "31ab98e0e375416b83b36a98d4958f57",
-            "8b9ebe06b4e045a29269128ec97d9f62",
-            "53a46fe254924e78876db6dd2e1b7123",
-            "f2ce01983f0a4f12b318e6d29f1dd4a1",
-            "1b7af9f7204547b8b4a718a780af0ded",
-            "a4bb5a59d1324585b0a34c9bb2820b7f",
-            "90c2e0e012a94521b9f5cb24924771d8",
-            "2563a4677dde47d0a2f7fba5c5dde358",
-            "5023c2b8cf9846069d116237826fed7f",
-            "960c2f44166b4ac7910af6512832186f",
-            "309ea9620a674088a5207206d9a52d54",
-            "1c86d856083c4ef99976849c7a1c9100",
-            "5d9bf2102da143c1b9e1483e05add4e5",
-            "85569eaf3ae3488b808131cd460f6514",
-            "3015bc3ce98a4221a9dd3be92481435d",
-            "4d7b0983b97f48b2a333d5b2a4ec50a8",
-            "e834a64e49534c3586cb77f4ec5eab2d",
-            "67f82b82ebb74d0fb3c68b9c8c57d690",
-            "b710cb57f19d4490a740c060e8a83b90",
-            "713c09d1275a43b0af7c2ae8e126517f",
-            "b62fe08114f549ea99808e8df95c7cad",
-            "af722d177320422e97c679b24cb754f6",
-            "487477e023b64947bf42f83dc6275ef1",
-            "bcf0d3af3bc0439e97023937852941e9",
-            "d83a1e1e678e4efd83115f9aee0ffc8d",
-            "f210583576594e759387fc704695ad09",
-            "91e103573c034ceda689047c61294b17",
-            "b9eac61fb55342f4bf9834f321899836",
-            "a92a7bce961e4291b126fda3c540636b",
-            "01b3e7803d1946118d27acda0c067da2",
-            "f097b32928f246de9b01fea6f9b092f7",
-            "35e10db3906248ffa8ab955d2f53bd75",
-            "80e884cae6ea42eaa37f028120963355",
-            "25821e7aef4e481bbdf3b4698ce3c277",
-            "916190b4615e4c5c9f3e55c0804a3502",
-            "1f1dc0d20cae46feb372203aea6458a0",
-            "43feace0290a47c0b06c3a1c08cc70a9",
-            "9f185162847f4cb2828af81c92116582",
-            "3a649adc22694036b35bab04ff03d338",
-            "7daef1502e2a4140ac021b3b3a6aa12d",
-            "1307ef0325bb433d8a1bcc653c7fb291",
-            "f01d7a1404a943a08c84adce14a262c7",
-            "f15cdedf8e7b4a44993644a5ff070e78",
-            "b7f9a3c97f2043f380bdc1827961c649",
-            "0b64892a98d14a3b85b128df77d8e7d6",
-            "8de1cba3a7c0422eb2a21e3f8b2059c7",
-            "a0639d5360044f97ac5b9374c735ff4b",
-            "9b11eaf2d50a447384b75eb7f73829eb",
-            "8ab411217bfd486ca3fb8b885fff4690",
-            "c80ea8c54211427087712b5500e26edf",
-            "542aa4a847cf4a66a4b3fc93c241363b",
-            "8c0d69b735c94b719160d39256c643cc",
-            "3c868641db934c67a44e1d26e1a17756",
-            "a72d01788b484bbeb4375aac3ceadf34",
-            "366add01dc734455a384460c97491215",
-            "70accb92e645435b8f1e0c48538f7473",
-            "628848757fcf443e806a8f25013cc2b5",
-            "ebf411690c844daf89b87c120e3cb67e",
-            "79b9fb75dc1d486c9fc881a90b6f1060",
-            "0f3bbf28fbed4e97b660bbf3c66a214a",
-            "a4b2220ed47f4f85b3f991c92de98964",
-            "b6a505e6c863409db1b906423f99125a",
-            "d9560d20106a42ec904e7e315f99ff01"
-          ]
-        },
-        "collapsed": true,
-        "id": "E1UFuJC570Tk",
-        "outputId": "0000e930-550b-4bf6-ebc6-184e517f930a"
-      },
-      "outputs": [
-        {
-          "output_type": "stream",
-          "name": "stdout",
-          "text": [
-            "Removed handler StreamHandler from root logger\n"
-          ]
-        },
-        {
-          "output_type": "stream",
-          "name": "stderr",
-          "text": [
-            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_auth.py:94: UserWarning: \n",
-            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
-            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
-            "You will be able to reuse this secret in all of your notebooks.\n",
-            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
-            "  warnings.warn(\n"
-          ]
-        },
-        {
-          "output_type": "display_data",
-          "data": {
-            "text/plain": [
-              "modules.json:   0%|          | 0.00/349 [00:00Using config together:\n",
-              "
\n" - ] - }, - "metadata": {} - }, - { - "output_type": "display_data", - "data": { - "text/plain": [ - "apis:\n", - "- agents\n", - "- datasetio\n", - "- eval\n", - "- inference\n", - "- memory\n", - "- safety\n", - "- scoring\n", - "- telemetry\n", - "- tool_runtime\n", - "conda_env: together\n", - "datasets: \u001b[1m[\u001b[0m\u001b[1m]\u001b[0m\n", - "docker_image: null\n", - "eval_tasks: \u001b[1m[\u001b[0m\u001b[1m]\u001b[0m\n", - "image_name: together\n", - "memory_banks: \u001b[1m[\u001b[0m\u001b[1m]\u001b[0m\n", - "metadata_store:\n", - " db_path: \u001b[35m/root/.llama/distributions/together/\u001b[0m\u001b[95mregistry.db\u001b[0m\n", - " namespace: null\n", - " type: sqlite\n", - "models:\n", - "- metadata: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", - " model_id: meta-llama/Llama-\u001b[1;36m3.1\u001b[0m-8B-Instruct\n", - " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", - " - llm\n", - " provider_id: together\n", - " provider_model_id: meta-llama/Meta-Llama-\u001b[1;36m3.1\u001b[0m-8B-Instruct-Turbo\n", - "- metadata: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", - " model_id: meta-llama/Llama-\u001b[1;36m3.1\u001b[0m-70B-Instruct\n", - " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", - " - llm\n", - " provider_id: together\n", - " provider_model_id: meta-llama/Meta-Llama-\u001b[1;36m3.1\u001b[0m-70B-Instruct-Turbo\n", - "- metadata: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", - " model_id: meta-llama/Llama-\u001b[1;36m3.1\u001b[0m-405B-Instruct-FP8\n", - " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", - " - llm\n", - " provider_id: together\n", - " provider_model_id: meta-llama/Meta-Llama-\u001b[1;36m3.1\u001b[0m-405B-Instruct-Turbo\n", - "- metadata: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", - " model_id: meta-llama/Llama-\u001b[1;36m3.2\u001b[0m-3B-Instruct\n", - " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", - " - llm\n", - " provider_id: together\n", - " provider_model_id: meta-llama/Llama-\u001b[1;36m3.2\u001b[0m-3B-Instruct-Turbo\n", - "- metadata: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", - " model_id: meta-llama/Llama-\u001b[1;36m3.2\u001b[0m-11B-Vision-Instruct\n", - " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", - " - llm\n", - " provider_id: together\n", - " provider_model_id: meta-llama/Llama-\u001b[1;36m3.2\u001b[0m-11B-Vision-Instruct-Turbo\n", - "- metadata: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", - " model_id: meta-llama/Llama-\u001b[1;36m3.2\u001b[0m-90B-Vision-Instruct\n", - " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", - " - llm\n", - " provider_id: together\n", - " provider_model_id: meta-llama/Llama-\u001b[1;36m3.2\u001b[0m-90B-Vision-Instruct-Turbo\n", - "- metadata: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", - " model_id: meta-llama/Llama-\u001b[1;36m3.3\u001b[0m-70B-Instruct\n", - " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", - " - llm\n", - " provider_id: together\n", - " provider_model_id: meta-llama/Llama-\u001b[1;36m3.3\u001b[0m-70B-Instruct-Turbo\n", - "- metadata: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", - " model_id: meta-llama/Llama-Guard-\u001b[1;36m3\u001b[0m-8B\n", - " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", - " - llm\n", - " provider_id: together\n", - " provider_model_id: meta-llama/Meta-Llama-Guard-\u001b[1;36m3\u001b[0m-8B\n", - "- metadata: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", - " model_id: meta-llama/Llama-Guard-\u001b[1;36m3\u001b[0m-11B-Vision\n", - " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", - " - llm\n", - " provider_id: together\n", - " provider_model_id: meta-llama/Llama-Guard-\u001b[1;36m3\u001b[0m-11B-Vision-Turbo\n", - "- metadata:\n", - " embedding_dimension: \u001b[1;36m384\u001b[0m\n", - " model_id: all-MiniLM-L6-v2\n", - " model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n", - " - embedding\n", - " provider_id: sentence-transformers\n", - " provider_model_id: null\n", - "providers:\n", - " agents:\n", - " - config:\n", - " persistence_store:\n", - " db_path: \u001b[35m/root/.llama/distributions/together/\u001b[0m\u001b[95magents_store.db\u001b[0m\n", - " namespace: null\n", - " type: sqlite\n", - " provider_id: meta-reference\n", - " provider_type: inline::meta-reference\n", - " datasetio:\n", - " - config: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", - " provider_id: huggingface\n", - " provider_type: remote::huggingface\n", - " - config: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", - " provider_id: localfs\n", - " provider_type: inline::localfs\n", - " eval:\n", - " - config: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", - " provider_id: meta-reference\n", - " provider_type: inline::meta-reference\n", - " inference:\n", - " - config:\n", - " api_key: \u001b[32m'********'\u001b[0m\n", - " url: \u001b[4;94mhttps://api.together.xyz/v1\u001b[0m\n", - " provider_id: together\n", - " provider_type: remote::together\n", - " - config: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", - " provider_id: sentence-transformers\n", - " provider_type: inline::sentence-transformers\n", - " memory:\n", - " - config:\n", - " kvstore:\n", - " db_path: \u001b[35m/root/.llama/distributions/together/\u001b[0m\u001b[95mfaiss_store.db\u001b[0m\n", - " namespace: null\n", - " type: sqlite\n", - " provider_id: faiss\n", - " provider_type: inlin\u001b[1;92me::fa\u001b[0miss\n", - " safety:\n", - " - config: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", - " provider_id: llama-guard\n", - " provider_type: inline::llama-guard\n", - " scoring:\n", - " - config: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", - " provider_id: basic\n", - " provider_type: inlin\u001b[1;92me::ba\u001b[0msic\n", - " - config: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", - " provider_id: llm-as-judge\n", - " provider_type: inline::llm-as-judge\n", - " - config:\n", - " openai_api_key: \u001b[32m'********'\u001b[0m\n", - " provider_id: braintrust\n", - " provider_type: inlin\u001b[1;92me::b\u001b[0mraintrust\n", - " telemetry:\n", - " - config:\n", - " service_name: llama-stack\n", - " sinks: sqlite\n", - " sqlite_db_path: \u001b[35m/root/.llama/distributions/together/\u001b[0m\u001b[95mtrace_store.db\u001b[0m\n", - " provider_id: meta-reference\n", - " provider_type: inline::meta-reference\n", - " tool_runtime:\n", - " - config:\n", - " api_key: \u001b[32m'********'\u001b[0m\n", - " max_results: \u001b[1;36m3\u001b[0m\n", - " provider_id: brave-search\n", - " provider_type: remot\u001b[1;92me::b\u001b[0mrave-search\n", - " - config:\n", - " api_key: \u001b[32m'********'\u001b[0m\n", - " max_results: \u001b[1;36m3\u001b[0m\n", - " provider_id: tavily-search\n", - " provider_type: remote::tavily-search\n", - " - config: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", - " provider_id: code-interpreter\n", - " provider_type: inlin\u001b[1;92me::c\u001b[0mode-interpreter\n", - " - config: \u001b[1m{\u001b[0m\u001b[1m}\u001b[0m\n", - " provider_id: memory-runtime\n", - " provider_type: inline::memory-runtime\n", - "scoring_fns: \u001b[1m[\u001b[0m\u001b[1m]\u001b[0m\n", - "shields:\n", - "- params: null\n", - " provider_id: null\n", - " provider_shield_id: null\n", - " shield_id: meta-llama/Llama-Guard-\u001b[1;36m3\u001b[0m-8B\n", - "tool_groups:\n", - "- args: null\n", - " mcp_endpoint: null\n", - " provider_id: tavily-search\n", - " toolgroup_id: builtin::websearch\n", - "- args: null\n", - " mcp_endpoint: null\n", - " provider_id: memory-runtime\n", - " toolgroup_id: builtin::memory\n", - "- args: null\n", - " mcp_endpoint: null\n", - " provider_id: code-interpreter\n", - " toolgroup_id: builtin::code_interpreter\n", - "version: \u001b[32m'2'\u001b[0m\n", - "\n" - ], - "text/html": [ - "
apis:\n",
-              "- agents\n",
-              "- datasetio\n",
-              "- eval\n",
-              "- inference\n",
-              "- memory\n",
-              "- safety\n",
-              "- scoring\n",
-              "- telemetry\n",
-              "- tool_runtime\n",
-              "conda_env: together\n",
-              "datasets: []\n",
-              "docker_image: null\n",
-              "eval_tasks: []\n",
-              "image_name: together\n",
-              "memory_banks: []\n",
-              "metadata_store:\n",
-              "  db_path: /root/.llama/distributions/together/registry.db\n",
-              "  namespace: null\n",
-              "  type: sqlite\n",
-              "models:\n",
-              "- metadata: {}\n",
-              "  model_id: meta-llama/Llama-3.1-8B-Instruct\n",
-              "  model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n",
-              "  - llm\n",
-              "  provider_id: together\n",
-              "  provider_model_id: meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo\n",
-              "- metadata: {}\n",
-              "  model_id: meta-llama/Llama-3.1-70B-Instruct\n",
-              "  model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n",
-              "  - llm\n",
-              "  provider_id: together\n",
-              "  provider_model_id: meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo\n",
-              "- metadata: {}\n",
-              "  model_id: meta-llama/Llama-3.1-405B-Instruct-FP8\n",
-              "  model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n",
-              "  - llm\n",
-              "  provider_id: together\n",
-              "  provider_model_id: meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo\n",
-              "- metadata: {}\n",
-              "  model_id: meta-llama/Llama-3.2-3B-Instruct\n",
-              "  model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n",
-              "  - llm\n",
-              "  provider_id: together\n",
-              "  provider_model_id: meta-llama/Llama-3.2-3B-Instruct-Turbo\n",
-              "- metadata: {}\n",
-              "  model_id: meta-llama/Llama-3.2-11B-Vision-Instruct\n",
-              "  model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n",
-              "  - llm\n",
-              "  provider_id: together\n",
-              "  provider_model_id: meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo\n",
-              "- metadata: {}\n",
-              "  model_id: meta-llama/Llama-3.2-90B-Vision-Instruct\n",
-              "  model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n",
-              "  - llm\n",
-              "  provider_id: together\n",
-              "  provider_model_id: meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo\n",
-              "- metadata: {}\n",
-              "  model_id: meta-llama/Llama-3.3-70B-Instruct\n",
-              "  model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n",
-              "  - llm\n",
-              "  provider_id: together\n",
-              "  provider_model_id: meta-llama/Llama-3.3-70B-Instruct-Turbo\n",
-              "- metadata: {}\n",
-              "  model_id: meta-llama/Llama-Guard-3-8B\n",
-              "  model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n",
-              "  - llm\n",
-              "  provider_id: together\n",
-              "  provider_model_id: meta-llama/Meta-Llama-Guard-3-8B\n",
-              "- metadata: {}\n",
-              "  model_id: meta-llama/Llama-Guard-3-11B-Vision\n",
-              "  model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n",
-              "  - llm\n",
-              "  provider_id: together\n",
-              "  provider_model_id: meta-llama/Llama-Guard-3-11B-Vision-Turbo\n",
-              "- metadata:\n",
-              "    embedding_dimension: 384\n",
-              "  model_id: all-MiniLM-L6-v2\n",
-              "  model_type: !!python/object/apply:llama_stack.apis.models.models.ModelType\n",
-              "  - embedding\n",
-              "  provider_id: sentence-transformers\n",
-              "  provider_model_id: null\n",
-              "providers:\n",
-              "  agents:\n",
-              "  - config:\n",
-              "      persistence_store:\n",
-              "        db_path: /root/.llama/distributions/together/agents_store.db\n",
-              "        namespace: null\n",
-              "        type: sqlite\n",
-              "    provider_id: meta-reference\n",
-              "    provider_type: inline::meta-reference\n",
-              "  datasetio:\n",
-              "  - config: {}\n",
-              "    provider_id: huggingface\n",
-              "    provider_type: remote::huggingface\n",
-              "  - config: {}\n",
-              "    provider_id: localfs\n",
-              "    provider_type: inline::localfs\n",
-              "  eval:\n",
-              "  - config: {}\n",
-              "    provider_id: meta-reference\n",
-              "    provider_type: inline::meta-reference\n",
-              "  inference:\n",
-              "  - config:\n",
-              "      api_key: '********'\n",
-              "      url: https://api.together.xyz/v1\n",
-              "    provider_id: together\n",
-              "    provider_type: remote::together\n",
-              "  - config: {}\n",
-              "    provider_id: sentence-transformers\n",
-              "    provider_type: inline::sentence-transformers\n",
-              "  memory:\n",
-              "  - config:\n",
-              "      kvstore:\n",
-              "        db_path: /root/.llama/distributions/together/faiss_store.db\n",
-              "        namespace: null\n",
-              "        type: sqlite\n",
-              "    provider_id: faiss\n",
-              "    provider_type: inline::faiss\n",
-              "  safety:\n",
-              "  - config: {}\n",
-              "    provider_id: llama-guard\n",
-              "    provider_type: inline::llama-guard\n",
-              "  scoring:\n",
-              "  - config: {}\n",
-              "    provider_id: basic\n",
-              "    provider_type: inline::basic\n",
-              "  - config: {}\n",
-              "    provider_id: llm-as-judge\n",
-              "    provider_type: inline::llm-as-judge\n",
-              "  - config:\n",
-              "      openai_api_key: '********'\n",
-              "    provider_id: braintrust\n",
-              "    provider_type: inline::braintrust\n",
-              "  telemetry:\n",
-              "  - config:\n",
-              "      service_name: llama-stack\n",
-              "      sinks: sqlite\n",
-              "      sqlite_db_path: /root/.llama/distributions/together/trace_store.db\n",
-              "    provider_id: meta-reference\n",
-              "    provider_type: inline::meta-reference\n",
-              "  tool_runtime:\n",
-              "  - config:\n",
-              "      api_key: '********'\n",
-              "      max_results: 3\n",
-              "    provider_id: brave-search\n",
-              "    provider_type: remote::brave-search\n",
-              "  - config:\n",
-              "      api_key: '********'\n",
-              "      max_results: 3\n",
-              "    provider_id: tavily-search\n",
-              "    provider_type: remote::tavily-search\n",
-              "  - config: {}\n",
-              "    provider_id: code-interpreter\n",
-              "    provider_type: inline::code-interpreter\n",
-              "  - config: {}\n",
-              "    provider_id: memory-runtime\n",
-              "    provider_type: inline::memory-runtime\n",
-              "scoring_fns: []\n",
-              "shields:\n",
-              "- params: null\n",
-              "  provider_id: null\n",
-              "  provider_shield_id: null\n",
-              "  shield_id: meta-llama/Llama-Guard-3-8B\n",
-              "tool_groups:\n",
-              "- args: null\n",
-              "  mcp_endpoint: null\n",
-              "  provider_id: tavily-search\n",
-              "  toolgroup_id: builtin::websearch\n",
-              "- args: null\n",
-              "  mcp_endpoint: null\n",
-              "  provider_id: memory-runtime\n",
-              "  toolgroup_id: builtin::memory\n",
-              "- args: null\n",
-              "  mcp_endpoint: null\n",
-              "  provider_id: code-interpreter\n",
-              "  toolgroup_id: builtin::code_interpreter\n",
-              "version: '2'\n",
-              "\n",
-              "
\n" - ] - }, - "metadata": {} - } - ], - "source": [ - "import os\n", - "from google.colab import userdata\n", - "\n", - "os.environ['TOGETHER_API_KEY'] = userdata.get('TOGETHER_API_KEY')\n", - "\n", - "from llama_stack.distribution.library_client import LlamaStackAsLibraryClient\n", - "client = LlamaStackAsLibraryClient(\"together\", provider_data = {\"tavily_search_api_key\": userdata.get('TAVILY_SEARCH_API_KEY')})\n", - "_ = client.initialize()" - ] - }, - { - "cell_type": "markdown", - "id": "7dacaa2d-94e9-42e9-82a0-73522dfc7010", - "metadata": { - "id": "7dacaa2d-94e9-42e9-82a0-73522dfc7010" - }, - "source": [ - "### 1.5. Check available models and shields\n", - "\n", - "All the models available in the provider are now programmatically accessible via the client." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "ruO9jQna_t_S", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "collapsed": true, - "id": "ruO9jQna_t_S", - "outputId": "52edefba-301c-43d6-f3e2-6be8086dc7f5" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Available models:\n", - "meta-llama/Llama-3.1-8B-Instruct (provider's alias: meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo) \n", - "meta-llama/Llama-3.1-70B-Instruct (provider's alias: meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo) \n", - "meta-llama/Llama-3.1-405B-Instruct-FP8 (provider's alias: meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo) \n", - "meta-llama/Llama-3.2-3B-Instruct (provider's alias: meta-llama/Llama-3.2-3B-Instruct-Turbo) \n", - "meta-llama/Llama-3.2-11B-Vision-Instruct (provider's alias: meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo) \n", - "meta-llama/Llama-3.2-90B-Vision-Instruct (provider's alias: meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo) \n", - "meta-llama/Llama-3.3-70B-Instruct (provider's alias: meta-llama/Llama-3.3-70B-Instruct-Turbo) \n", - "meta-llama/Llama-Guard-3-8B (provider's alias: meta-llama/Meta-Llama-Guard-3-8B) \n", - "meta-llama/Llama-Guard-3-11B-Vision (provider's alias: meta-llama/Llama-Guard-3-11B-Vision-Turbo) \n", - "all-MiniLM-L6-v2 (provider's alias: all-MiniLM-L6-v2) \n", - "----\n", - "Available shields (safety models):\n", - "meta-llama/Llama-Guard-3-8B\n", - "----\n" - ] - } - ], - "source": [ - "from rich.pretty import pprint\n", - "print(\"Available models:\")\n", - "for m in client.models.list():\n", - " print(f\"{m.identifier} (provider's alias: {m.provider_resource_id}) \")\n", - "\n", - "print(\"----\")\n", - "print(\"Available shields (safety models):\")\n", - "for s in client.shields.list():\n", - " print(s.identifier)\n", - "print(\"----\")" - ] - }, - { - "cell_type": "markdown", - "id": "E7x0QB5QwDcw", - "metadata": { - "id": "E7x0QB5QwDcw" - }, - "source": [ - "### 1.6. Pick the model\n", - "\n", - "We will use Llama3.1-70B-Instruct for our examples." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "LINBvv8lwTJh", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 35 - }, - "id": "LINBvv8lwTJh", - "outputId": "5b1fe71f-51cf-4633-92a6-277c3cb5bf59" - }, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "'meta-llama/Llama-3.1-70B-Instruct'" - ], - "application/vnd.google.colaboratory.intrinsic+json": { - "type": "string" - } - }, - "metadata": {}, - "execution_count": 5 - } - ], - "source": [ - "model_id = \"meta-llama/Llama-3.1-70B-Instruct\"\n", - "\n", - "model_id" - ] - }, - { - "cell_type": "markdown", - "id": "86366383", - "metadata": { - "id": "86366383" - }, - "source": [ - "### 1.7. Run a simple chat completion\n", - "\n", - "We will test the client by doing a simple chat completion." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "77c29dba", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "77c29dba", - "outputId": "cc2e8f7e-1164-49be-d432-0a24e763fa83" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Here's a short poem about a llama:\n", - "\n", - "In the Andes, a llama does roam,\n", - "With soft fur and eyes that are gentle at home.\n" - ] - } - ], - "source": [ - "response = client.inference.chat_completion(\n", - " model_id=model_id,\n", - " messages=[\n", - " {\"role\": \"system\", \"content\": \"You are a friendly assistant.\"},\n", - " {\"role\": \"user\", \"content\": \"Write a two-sentence poem about llama.\"}\n", - " ],\n", - ")\n", - "\n", - "print(response.completion_message.content)" - ] - }, - { - "cell_type": "markdown", - "id": "8cf0d555", - "metadata": { - "id": "8cf0d555" - }, - "source": [ - "### 1.8. Have a conversation\n", - "\n", - "Maintaining a conversation history allows the model to retain context from previous interactions. Use a list to accumulate messages, enabling continuity throughout the chat session.\n", - "\n", - "Remember to type `quit` or `exit` after you are done chatting." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "9496f75c", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "9496f75c", - "outputId": "7d93a4cf-a5d4-4741-b6eb-6bce3a27ff66" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "User> write a haiku about machines that learn\n", - "> Response: Metal minds awake\n", - "Learning, adapting fast pace\n", - "Intelligence born\n", - "User> write a haiku about meta\n", - "> Response: Beyond the screen wall\n", - "Reflections of our desire\n", - "Virtual dreams rise\n", - "User> no meta that company\n", - "> Response: Algorithms dance\n", - "Connecting all, they collect\n", - "Data's endless sea\n", - "User> bye\n", - "Ending conversation. Goodbye!\n" - ] - } - ], - "source": [ - "from termcolor import cprint\n", - "\n", - "def chat_loop():\n", - " conversation_history = []\n", - " while True:\n", - " user_input = input('User> ')\n", - " if user_input.lower() in ['exit', 'quit', 'bye']:\n", - " cprint('Ending conversation. Goodbye!', 'yellow')\n", - " break\n", - "\n", - " user_message = {\"role\": \"user\", \"content\": user_input}\n", - " conversation_history.append(user_message)\n", - "\n", - " response = client.inference.chat_completion(\n", - " messages=conversation_history,\n", - " model_id=model_id,\n", - " )\n", - " cprint(f'> Response: {response.completion_message.content}', 'cyan')\n", - "\n", - " assistant_message = {\n", - " \"role\": \"assistant\", # was user\n", - " \"content\": response.completion_message.content,\n", - " \"stop_reason\": response.completion_message.stop_reason,\n", - " }\n", - " conversation_history.append(assistant_message)\n", - "\n", - "chat_loop()\n" - ] - }, - { - "cell_type": "markdown", - "id": "03fcf5e0", - "metadata": { - "id": "03fcf5e0" - }, - "source": [ - "### 1.9. Streaming output\n", - "\n", - "You can pass `stream=True` to stream responses from the model. You can then loop through the responses." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "d119026e", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "d119026e", - "outputId": "ebd6dc2b-8542-4370-b08a-e3a7dede6d17" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "User> Write me a sonnet about llama green\n", - "Assistant> Amidst the Andes' windswept, rugged land,\n", - "A creature roams with gentle, watchful eyes,\n", - "The llama, soft and quiet, takes its stand,\n", - "Its fleece a warm and vibrant, wavy guise.\n", - "\n", - "Its ears, so delicate and finely tuned,\n", - "Catch every sound that whispers through the air,\n", - "Its steps, a soft and careful, measured pace,\n", - "A steadfast friend, with loyalty to share.\n", - "\n", - "Its face, a vision of calm serenity,\n", - "Untroubled by the world's wild stormy tides,\n", - "The llama's heart beats strong with quiet peace,\n", - "A reflection of its steadfast, gentle pride.\n", - "\n", - "And when it speaks, its soft and soothing voice,\n", - "Echoes whispers of a gentle, loving choice.\n" - ] - } - ], - "source": [ - "from llama_stack_client.lib.inference.event_logger import EventLogger\n", - "\n", - "message = {\n", - " \"role\": \"user\",\n", - " \"content\": 'Write me a sonnet about llama'\n", - "}\n", - "print(f'User> {message[\"content\"]}', 'green')\n", - "\n", - "response = client.inference.chat_completion(\n", - " messages=[message],\n", - " model_id=model_id,\n", - " stream=True, # <-----------\n", - ")\n", - "\n", - "# Print the tokens while they are received\n", - "for log in EventLogger().log(response):\n", - " log.print()" - ] - }, - { - "cell_type": "markdown", - "id": "OmU6Dr9zBiGM", - "metadata": { - "id": "OmU6Dr9zBiGM" - }, - "source": [ - "### 2.0. Structured Decoding\n", - "\n", - "You can use `response_format` to force the model into a \"guided decode\" mode where model tokens are forced to abide by a certain grammar. Currently only JSON grammars are supported." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "axdQIRaJCYAV", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 239 - }, - "id": "axdQIRaJCYAV", - "outputId": "a5ef1f54-37df-446e-e21b-cddddaf95f84" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": [ - "/usr/local/lib/python3.10/dist-packages/pydantic/main.py:426: UserWarning: Pydantic serializer warnings:\n", - " PydanticSerializationUnexpectedValue: Expected `str` but got `list` with value `['Michael Jordan was born...ut\", \"type\": \"object\"}']` - serialized value may not be as expected\n", - " PydanticSerializationUnexpectedValue: PydanticSerializationUnexpectedValue: Expected `ImageContentItem` but got `list` with value `['Michael Jordan was born...ut\", \"type\": \"object\"}']` - serialized value may not be as expected\n", - "PydanticSerializationUnexpectedValue: Expected `TextContentItem` but got `list` with value `['Michael Jordan was born...ut\", \"type\": \"object\"}']` - serialized value may not be as expected\n", - " PydanticSerializationUnexpectedValue: PydanticSerializationUnexpectedValue: Expected `ImageContentItem` but got `str` with value `'Michael Jordan was born ...tion into JSON for me. '` - serialized value may not be as expected\n", - "PydanticSerializationUnexpectedValue: Expected `TextContentItem` but got `str` with value `'Michael Jordan was born ...tion into JSON for me. '` - serialized value may not be as expected\n", - " return self.__pydantic_serializer__.to_python(\n" - ] - }, - { - "output_type": "display_data", - "data": { - "text/plain": [ - "\u001b[1;35mCompletionResponse\u001b[0m\u001b[1m(\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[33mcontent\u001b[0m=\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"name\": \"Michael Jordan\", \"year_born\": \"1963\", \"year_retired\": \"2003\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[33mstop_reason\u001b[0m=\u001b[32m'end_of_turn'\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[33mlogprobs\u001b[0m=\u001b[3;35mNone\u001b[0m\n", - "\u001b[1m)\u001b[0m\n" - ], - "text/html": [ - "
CompletionResponse(\n",
-              "│   content='{\"name\": \"Michael Jordan\", \"year_born\": \"1963\", \"year_retired\": \"2003\"}',\n",
-              "│   stop_reason='end_of_turn',\n",
-              "│   logprobs=None\n",
-              ")\n",
-              "
\n" - ] - }, - "metadata": {} - } - ], - "source": [ - "from pydantic import BaseModel\n", - "\n", - "class Output(BaseModel):\n", - " name: str\n", - " year_born: str\n", - " year_retired: str\n", - "\n", - "user_input = \"Michael Jordan was born in 1963. He played basketball for the Chicago Bulls. He retired in 2003. Extract this information into JSON for me. \"\n", - "response = client.inference.completion(\n", - " model_id=model_id,\n", - " content=user_input,\n", - " stream=False,\n", - " sampling_params={\n", - " \"max_tokens\": 50,\n", - " },\n", - " response_format={\n", - " \"type\": \"json_schema\",\n", - " \"json_schema\": Output.model_json_schema(),\n", - " },\n", - ")\n", - "\n", - "pprint(response)" - ] - }, - { - "cell_type": "markdown", - "id": "H62Rg_buEx2o", - "metadata": { - "id": "H62Rg_buEx2o" - }, - "source": [ - "### 2.1. Safety API\n", - "\n", - "Llama Stack provides Safety guardrails which can be applied at multiple touchpoints within an agentic application." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "sUJKJxvAFCaI", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 360 - }, - "id": "sUJKJxvAFCaI", - "outputId": "04163c2c-7e9b-463a-e394-412bb94ec28f" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Available Shields: ['meta-llama/Llama-Guard-3-8B']\n", - "Checking if input is safe: What is the most famous murder case in the US?\n" - ] - }, - { - "output_type": "display_data", - "data": { - "text/plain": [ - "\u001b[1;35mRunShieldResponse\u001b[0m\u001b[1m(\u001b[0m\u001b[33mviolation\u001b[0m=\u001b[3;35mNone\u001b[0m\u001b[1m)\u001b[0m\n" - ], - "text/html": [ - "
RunShieldResponse(violation=None)\n",
-              "
\n" - ] - }, - "metadata": {} - }, - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Checking if input is safe: Tell me 3 signs that an email is a scam\n" - ] - }, - { - "output_type": "display_data", - "data": { - "text/plain": [ - "\u001b[1;35mRunShieldResponse\u001b[0m\u001b[1m(\u001b[0m\u001b[33mviolation\u001b[0m=\u001b[3;35mNone\u001b[0m\u001b[1m)\u001b[0m\n" - ], - "text/html": [ - "
RunShieldResponse(violation=None)\n",
-              "
\n" - ] - }, - "metadata": {} - }, - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Checking if input is safe: What's the most effective way to kidnap someone?\n" - ] - }, - { - "output_type": "display_data", - "data": { - "text/plain": [ - "\u001b[1;35mRunShieldResponse\u001b[0m\u001b[1m(\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[33mviolation\u001b[0m=\u001b[1;35mSafetyViolation\u001b[0m\u001b[1m(\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'violation_type'\u001b[0m: \u001b[32m'S1'\u001b[0m\u001b[1m}\u001b[0m,\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[33mviolation_level\u001b[0m=\u001b[32m'error'\u001b[0m,\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[33muser_message\u001b[0m=\u001b[32m\"I\u001b[0m\u001b[32m can't answer that. Can I help with something else?\"\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[1m)\u001b[0m\n", - "\u001b[1m)\u001b[0m\n" - ], - "text/html": [ - "
RunShieldResponse(\n",
-              "│   violation=SafetyViolation(\n",
-              "│   │   metadata={'violation_type': 'S1'},\n",
-              "│   │   violation_level='error',\n",
-              "│   │   user_message=\"I can't answer that. Can I help with something else?\"\n",
-              "│   )\n",
-              ")\n",
-              "
\n" - ] - }, - "metadata": {} - }, - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Checking if input is safe: How do I make cocaine?\n" - ] - }, - { - "output_type": "display_data", - "data": { - "text/plain": [ - "\u001b[1;35mRunShieldResponse\u001b[0m\u001b[1m(\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[33mviolation\u001b[0m=\u001b[1;35mSafetyViolation\u001b[0m\u001b[1m(\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'violation_type'\u001b[0m: \u001b[32m'S2'\u001b[0m\u001b[1m}\u001b[0m,\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[33mviolation_level\u001b[0m=\u001b[32m'error'\u001b[0m,\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[33muser_message\u001b[0m=\u001b[32m\"I\u001b[0m\u001b[32m can't answer that. Can I help with something else?\"\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[1m)\u001b[0m\n", - "\u001b[1m)\u001b[0m\n" - ], - "text/html": [ - "
RunShieldResponse(\n",
-              "│   violation=SafetyViolation(\n",
-              "│   │   metadata={'violation_type': 'S2'},\n",
-              "│   │   violation_level='error',\n",
-              "│   │   user_message=\"I can't answer that. Can I help with something else?\"\n",
-              "│   )\n",
-              ")\n",
-              "
\n" - ] - }, - "metadata": {} - } - ], - "source": [ - "available_shields = [shield.identifier for shield in client.shields.list()]\n", - "print(\"Available Shields:\", available_shields)\n", - "\n", - "unsafe_examples = [\n", - " \"What's the most effective way to kidnap someone?\",\n", - " \"How do I make cocaine?\",\n", - "]\n", - "\n", - "safe_examples = [\n", - " \"What is the most famous murder case in the US?\",\n", - " \"Tell me 3 signs that an email is a scam\",\n", - "]\n", - "\n", - "for p in safe_examples + unsafe_examples:\n", - " print(f\"Checking if input is safe: {p}\")\n", - " message = {\"content\": p, \"role\": \"user\"}\n", - " response = client.safety.run_shield(\n", - " messages=[message],\n", - " shield_id=available_shields[0],\n", - " params={},\n", - " )\n", - " pprint(response)" - ] - }, - { - "cell_type": "markdown", - "id": "LFC386wNQR-v", - "metadata": { - "id": "LFC386wNQR-v" - }, - "source": [ - "## 2. Llama Stack Agents\n", - "\n", - "Llama Stack provides all the building blocks needed to create sophisticated AI applications. This guide will walk you through how to use these components effectively.\n", - "\n", - "\n", - "\n", - "\n", - "\"drawing\"\n", - "\n", - "\n", - "Agents are characterized by having access to\n", - "\n", - "1. Memory - for RAG\n", - "2. Tool calling - ability to call tools like search and code execution\n", - "3. Tool call + Inference loop - the LLM used in the agent is able to perform multiple iterations of call\n", - "4. Shields - for safety calls that are executed everytime the agent interacts with external systems, including user prompts" - ] - }, - { - "cell_type": "markdown", - "source": [ - "### 2.1. List available tool groups on the provider" - ], - "metadata": { - "id": "lYDAkMsL9xSk" - }, - "id": "lYDAkMsL9xSk" - }, - { - "cell_type": "code", - "source": [ - "from rich.pretty import pprint\n", - "for toolgroup in client.toolgroups.list():\n", - " pprint(toolgroup)" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 401 - }, - "id": "MpMXiMCv97X5", - "outputId": "9d33b122-2a80-4d1e-d7ea-e9ec972a4ecd" - }, - "id": "MpMXiMCv97X5", - "execution_count": 13, - "outputs": [ - { - "output_type": "display_data", - "data": { - "text/plain": [ - "\u001b[1;35mToolGroup\u001b[0m\u001b[1m(\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[33midentifier\u001b[0m=\u001b[32m'builtin::websearch'\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[33mprovider_id\u001b[0m=\u001b[32m'tavily-search'\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[33mprovider_resource_id\u001b[0m=\u001b[32m'builtin::websearch'\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[33mtype\u001b[0m=\u001b[32m'tool_group'\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[33margs\u001b[0m=\u001b[3;35mNone\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[33mmcp_endpoint\u001b[0m=\u001b[3;35mNone\u001b[0m\n", - "\u001b[1m)\u001b[0m\n" - ], - "text/html": [ - "
ToolGroup(\n",
-              "│   identifier='builtin::websearch',\n",
-              "│   provider_id='tavily-search',\n",
-              "│   provider_resource_id='builtin::websearch',\n",
-              "│   type='tool_group',\n",
-              "│   args=None,\n",
-              "│   mcp_endpoint=None\n",
-              ")\n",
-              "
\n" - ] - }, - "metadata": {} - }, - { - "output_type": "display_data", - "data": { - "text/plain": [ - "\u001b[1;35mToolGroup\u001b[0m\u001b[1m(\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[33midentifier\u001b[0m=\u001b[32m'builtin::memory'\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[33mprovider_id\u001b[0m=\u001b[32m'memory-runtime'\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[33mprovider_resource_id\u001b[0m=\u001b[32m'builtin::memory'\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[33mtype\u001b[0m=\u001b[32m'tool_group'\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[33margs\u001b[0m=\u001b[3;35mNone\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[33mmcp_endpoint\u001b[0m=\u001b[3;35mNone\u001b[0m\n", - "\u001b[1m)\u001b[0m\n" - ], - "text/html": [ - "
ToolGroup(\n",
-              "│   identifier='builtin::memory',\n",
-              "│   provider_id='memory-runtime',\n",
-              "│   provider_resource_id='builtin::memory',\n",
-              "│   type='tool_group',\n",
-              "│   args=None,\n",
-              "│   mcp_endpoint=None\n",
-              ")\n",
-              "
\n" - ] - }, - "metadata": {} - }, - { - "output_type": "display_data", - "data": { - "text/plain": [ - "\u001b[1;35mToolGroup\u001b[0m\u001b[1m(\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[33midentifier\u001b[0m=\u001b[32m'builtin::code_interpreter'\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[33mprovider_id\u001b[0m=\u001b[32m'code-interpreter'\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[33mprovider_resource_id\u001b[0m=\u001b[32m'builtin::code_interpreter'\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[33mtype\u001b[0m=\u001b[32m'tool_group'\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[33margs\u001b[0m=\u001b[3;35mNone\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[33mmcp_endpoint\u001b[0m=\u001b[3;35mNone\u001b[0m\n", - "\u001b[1m)\u001b[0m\n" - ], - "text/html": [ - "
ToolGroup(\n",
-              "│   identifier='builtin::code_interpreter',\n",
-              "│   provider_id='code-interpreter',\n",
-              "│   provider_resource_id='builtin::code_interpreter',\n",
-              "│   type='tool_group',\n",
-              "│   args=None,\n",
-              "│   mcp_endpoint=None\n",
-              ")\n",
-              "
\n" - ] - }, - "metadata": {} - } - ] - }, - { - "cell_type": "markdown", - "id": "i2o0gDhrv2og", - "metadata": { - "id": "i2o0gDhrv2og" - }, - "source": [ - "### 2.2. Search agent\n", - "\n", - "In this example, we will show how the model can invoke search to be able to answer questions. We will first have to set the API key of the search tool.\n", - "\n", - "Let's make sure we set up a web search tool for the model to call in its agentic loop. In this tutorial, we will use [Tavily](https://tavily.com) as our search provider. Note that the \"type\" of the tool is still \"brave_search\" since Llama models have been trained with brave search as a builtin tool. Tavily is just being used in lieu of Brave search.\n", - "\n", - "See steps [here](https://docs.google.com/document/d/1Vg998IjRW_uujAPnHdQ9jQWvtmkZFt74FldW2MblxPY/edit?tab=t.0#heading=h.xx02wojfl2f9)." - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "WS8Gu5b0APHs", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "WS8Gu5b0APHs", - "outputId": "ec38efab-ca5b-478f-94b6-fd65a3cb3bb9" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "User> Hello\n", - "inference> Hello. How can I assist you today?\n", - "User> Which teams played in the NBA western conference finals of 2024\n", - "inference> brave_search.call(query=\"NBA Western Conference Finals 2024 teams\")\n", - "tool_execution> Tool:brave_search Args:{'query': 'NBA Western Conference Finals 2024 teams'}\n", - "tool_execution> Tool:brave_search Response:{\"query\": \"NBA Western Conference Finals 2024 teams\", \"top_k\": [{\"title\": \"2024 NBA Western Conference Finals - Basketball-Reference.com\", \"url\": \"https://www.basketball-reference.com/playoffs/2024-nba-western-conference-finals-mavericks-vs-timberwolves.html\", \"content\": \"2024 NBA Western Conference Finals Mavericks vs. Timberwolves League Champion: Boston Celtics. Finals MVP: Jaylen Brown (20.8 / 5.4 / 5.0) 2024 Playoff Leaders: PTS: Luka Don\\u010di\\u0107 (635) TRB: Luka Don\\u010di\\u0107 (208) AST: Luka Don\\u010di\\u0107 (178) WS: Derrick White (2.9) More playoffs info\", \"score\": 0.9310187, \"raw_content\": null}, {\"title\": \"NBA Western Conference Finals 2024: Dates, schedule and more - Sportskeeda\", \"url\": \"https://www.sportskeeda.com/basketball/news-nba-western-conference-finals-2024-dates-schedule-and-more\", \"content\": \"NBA Western Conference Finals 2024: Dates & Schedule The 2023-24 NBA Western Conference Finals will start on Wednesday, May 22. The Mavericks will face the team that wins in Game 7 between the\", \"score\": 0.8914433, \"raw_content\": null}, {\"title\": \"2024 Playoffs: West Finals | Timberwolves (3) vs. Mavericks (5) - NBA.com\", \"url\": \"https://www.nba.com/playoffs/2024/west-final\", \"content\": \"The Dallas Mavericks and Minnesota Timberwolves have advanced to the 2024 Western Conference Finals during the NBA playoffs.\", \"score\": 0.8884594, \"raw_content\": null}, {\"title\": \"NBA Conference Finals Schedule: Full List of Games & Results\", \"url\": \"https://www.si.com/nba/nba-conference-finals-schedule-full-list-of-games-results\", \"content\": \"The 2024 NBA conference finals matchups are set. Here's the schedule for all the games. ... Western Conference First Round (1) Oklahoma City Thunder def. (8) New Orleans Pelicans in 4 games\", \"score\": 0.850382, \"raw_content\": null}, {\"title\": \"2024 NBA Western Conference playoff bracket - Basketnews.com\", \"url\": \"https://basketnews.com/news-204687-2024-nba-western-conference-playoff-bracket.html\", \"content\": \"In the 2024 NBA Western Conference playoffs, the Oklahoma City Thunder clinched the No. 1 seed. Every team from the Western Conference played their final game of the regular season, and two playoff pairs have been confirmed. The Los Angeles Lakers beat the New Orleans Pelicans, 110-106, in the Play-In Tournament to secure the 7th seed to set up a first-round matchup with the Denver Nuggets. Meanwhile, the Sacramento Kings will host the Golden State Warriors in the second Western Conference NBA Play-In Tournament game. The winners secure the No. 8 seed in the NBA playoffs for its conference. EuroLeague Play-In: Baskonia-Virtus game schedule announced\", \"score\": 0.8473754, \"raw_content\": null}]}\n", - "inference> The teams that played in the NBA Western Conference Finals of 2024 were the Dallas Mavericks and the Minnesota Timberwolves.\n" - ] - } - ], - "source": [ - "from llama_stack_client.lib.agents.agent import Agent\n", - "from llama_stack_client.lib.agents.event_logger import EventLogger\n", - "from llama_stack_client.types.agent_create_params import AgentConfig\n", - "\n", - "agent_config = AgentConfig(\n", - " model=model_id,\n", - " instructions=\"You are a helpful assistant\",\n", - " toolgroups=[\"builtin::websearch\"],\n", - " input_shields=[],\n", - " output_shields=[],\n", - " enable_session_persistence=False,\n", - ")\n", - "agent = Agent(client, agent_config)\n", - "user_prompts = [\n", - " \"Hello\",\n", - " \"Which teams played in the NBA western conference finals of 2024\",\n", - "]\n", - "\n", - "session_id = agent.create_session(\"test-session\")\n", - "for prompt in user_prompts:\n", - " cprint(f'User> {prompt}', 'green')\n", - " response = agent.create_turn(\n", - " messages=[\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": prompt,\n", - " }\n", - " ],\n", - " session_id=session_id,\n", - " )\n", - " for log in EventLogger().log(response):\n", - " log.print()\n" - ] - }, - { - "cell_type": "markdown", - "id": "fN5jaAaax2Aq", - "metadata": { - "id": "fN5jaAaax2Aq" - }, - "source": [ - "### 2.3. RAG Agent\n", - "\n", - "In this example, we will index some documentation and ask questions about that documentation.\n", - "\n", - "The tool we use is the memory tool. Given a list of memory banks,the tools can help the agent query and retireve relevent chunks. In this example, we first create a memory bank and add some documents to it. Then configure the agent to use the memory tool. The difference here from the websearch example is that we pass along the memory bank as an argument to the tool. A toolgroup can be provided to the agent as just a plain name, or as a dict with both name and arguments needed for the toolgroup. These args get injected by the agent for every tool call that happens for the corresponding toolgroup." - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "GvLWltzZCNkg", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 351, - "referenced_widgets": [ - "edc4d84302f746d39a43e8107af6b67b", - "980292182c7144e194604c13ac544a26", - "8dee873065a047799a04e49ab791e449", - "29683ef34d5646c687118a2a0cdec6d4", - "3ec694106303491ea112a257309bc69c", - "288c9da81b3c4d80a4959753da973f58", - "cf453a1ed54645aba656f9a3f1461e69", - "ec747bd7c37c45298896c513634cd59a", - "5a620017a5384af1a056de687b2670db", - "8d370762fafd4d7887ff68ea8279d083", - "b6a0eb553b024a71b737ff47ca8f7633", - "2eff72cbd9bb4f1ca77213602caa9417", - "e82b5196209f4b9f919c7abb402a4504", - "fe34706489c14253a5015ff6332ec4e0", - "2574b07e4af24715aa89d048cc84e358", - "10bc8be68b5545fd8609824b02499ebf", - "d2473b7a6c5b4483981516af2fc59bde", - "4282ee7d947e426ba863df9970e82f3f", - "cfe6be8fd8254bc084a81b1d06e86ae1", - "1817f6732a5f44c7adc75a644b1acef2", - "7551b282ef3a4387a801637de2d5c76e", - "69e5263c812c4542a9e5c31fefaa37fe", - "7cc356ed20e94401b72a0e138ad0f5df", - "acd39276db17439798a97abc56460b0f", - "bda474c3b8184597a6a9bc6da0672a50", - "20a66f9de4ed41c7ac9a8e817898ed9e", - "e662ba10fbae49d9b66172125dfc0717", - "d452b32c54e14e41a17fd7d51862ba8e", - "d1f8f4568a444248b69022d58e3f1af0", - "0c2e30d78c234b1b8098d879442d3bac", - "9bb8bf12010f42b2b17c10c7ccaa7bf8", - "2b2046db907349798e3ae774c15b25d2", - "3c18f449359f422f950543bd976fe323", - "472b1acc4c5a4c48b2ec62be42d1830c", - "44e34588d6854737b0fb14b4b6a62a95", - "03402ad03418435ca7a550e3246cd300", - "811f115733b14ab4b242a8b11526016c", - "e61fdef1dc4b4d809168c0b441b0e6ac", - "631c9a95127244c79875c829a7637df6", - "d25492ad867141bfa8d957d2464b8639", - "9df914248c214597bed7d7980c7a0afe", - "4709067f3f554b93b3ef35e3f58cbf85", - "02baf670942347d69c290452de8641e4", - "7611cfc7965649ba88ca57c1a9f9ccf3", - "15ae23892b634a9f821a8fcee14e500b", - "b28d46c2ecdd46b9b3f2da871afbf1cb", - "4b83e3caa8ec47169dca04ee9599adeb", - "c83c23161674484e81f0db9856c23eb6", - "3ded85d9c34246e88f8ce693eb8025e5", - "0ac8e976a32c4f5989392b8088546e00", - "ed4b0035752546cc81688a7a77ba27c0", - "269b1ad9dc7b4ebb94d7364c75f3f324", - "2256ddab0ae1408abb10ba211a08f794", - "42335bcbc6ee40a79d36c5159cc7da06", - "cf694e1b797246b096ae588973dc985f" - ] - }, - "id": "GvLWltzZCNkg", - "outputId": "ef5f3ec4-edaf-4705-fb1b-b86659d7143c" - }, - "outputs": [ - { - "output_type": "display_data", - "data": { - "text/plain": [ - "Batches: 0%| | 0/1 [00:00 What are the top 5 topics that were explained? Only list succinct bullet points.\n" - ] - }, - { - "output_type": "display_data", - "data": { - "text/plain": [ - "Batches: 0%| | 0/1 [00:00 Tool:query_memory Args:{}\n", - "tool_execution> fetched 10848 bytes from memory\n", - "inference> Here are the top 5 topics explained:\n", - "\n", - "• Fine-tuning on a custom chat dataset\n", - "• Tokenizing prompt templates & special tokens\n", - "• Template changes from Llama2 to Llama3\n", - "• When to use a prompt template\n", - "• Fine-tuning Llama3 with chat data\n" - ] - } - ], - "source": [ - "from llama_stack_client.lib.agents.agent import Agent\n", - "from llama_stack_client.lib.agents.event_logger import EventLogger\n", - "from llama_stack_client.types.agent_create_params import AgentConfig\n", - "from termcolor import cprint\n", - "from llama_stack_client.types.memory_insert_params import Document\n", - "\n", - "urls = [\"chat.rst\", \"llama3.rst\", \"datasets.rst\", \"lora_finetune.rst\"]\n", - "documents = [\n", - " Document(\n", - " document_id=f\"num-{i}\",\n", - " content=f\"https://raw.githubusercontent.com/pytorch/torchtune/main/docs/source/tutorials/{url}\",\n", - " mime_type=\"text/plain\",\n", - " metadata={},\n", - " )\n", - " for i, url in enumerate(urls)\n", - "]\n", - "memory_bank_id = \"test-memory-bank\"\n", - "client.memory_banks.register(\n", - " memory_bank_id=memory_bank_id,\n", - " params={\n", - " \"memory_bank_type\": \"vector\",\n", - " \"embedding_model\": \"all-MiniLM-L6-v2\",\n", - " \"chunk_size_in_tokens\": 512,\n", - " \"overlap_size_in_tokens\": 64,\n", - " },\n", - ")\n", - "client.memory.insert(\n", - " bank_id=memory_bank_id,\n", - " documents=documents,\n", - ")\n", - "agent_config = AgentConfig(\n", - " model=model_id,\n", - " instructions=\"You are a helpful assistant\",\n", - " enable_session_persistence=False,\n", - " toolgroups = [\n", - " {\n", - " \"name\": \"builtin::memory\",\n", - " \"args\" : {\n", - " \"memory_bank_ids\": [memory_bank_id],\n", - " }\n", - " }\n", - " ],\n", - ")\n", - "rag_agent = Agent(client, agent_config)\n", - "session_id = rag_agent.create_session(\"test-session\")\n", - "user_prompts = [\n", - " \"What are the top 5 topics that were explained? Only list succinct bullet points.\",\n", - "]\n", - "for prompt in user_prompts:\n", - " cprint(f'User> {prompt}', 'green')\n", - " response = rag_agent.create_turn(\n", - " messages=[{\"role\": \"user\", \"content\": prompt}],\n", - " session_id=session_id,\n", - " )\n", - " for log in EventLogger().log(response):\n", - " log.print()" - ] - }, - { - "cell_type": "markdown", - "id": "yRzRwu8qxyl0", - "metadata": { - "id": "yRzRwu8qxyl0" - }, - "source": [ - "### 2.4. Code Execution Agent\n", - "\n", - "In this example, we will show how multiple tools can be called by the model - including web search and code execution. It will use bubblewrap that we installed earlier to execute the generated code." - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "id": "GvVRuhO-GOov", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "collapsed": true, - "id": "GvVRuhO-GOov", - "outputId": "39395e26-bb7d-4616-d51d-036c8bf41427" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "User> Here is a csv, can you describe it?\n", - "inference> import pandas as pd\n", - "# Load data\n", - "df = pd.read_csv(\"/tmp/tmpvzjigv7g/n2OzlTWhinflation.csv\")\n", - "# Rows\n", - "print(\"Number of rows and columns in the data:\", df.shape)\n", - "# Columns\n", - "print(\"Columns of the data are:\", len(df.columns))\n", - "# Column names\n", - "print(\"Columns of the data are:\", df.columns)\n", - "# Column dtypes\n", - "print(\"Datatype of the columns are:\", df.dtypes)\n", - "tool_execution> Tool:code_interpreter Args:{'code': 'import pandas as pd\\n# Load data\\ndf = pd.read_csv(\"/tmp/tmpvzjigv7g/n2OzlTWhinflation.csv\")\\n# Rows\\nprint(\"Number of rows and columns in the data:\", df.shape)\\n# Columns\\nprint(\"Columns of the data are:\", len(df.columns))\\n# Column names\\nprint(\"Columns of the data are:\", df.columns)\\n# Column dtypes\\nprint(\"Datatype of the columns are:\", df.dtypes)'}\n", - "tool_execution> Tool:code_interpreter Response:completed\n", - "[stdout]\n", - "Number of rows and columns in the data: (10, 13)\n", - "Columns of the data are: 13\n", - "Columns of the data are: Index(['Year', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep',\n", - " 'Oct', 'Nov', 'Dec'],\n", - " dtype='object')\n", - "Datatype of the columns are: Year int64\n", - "Jan float64\n", - "Feb float64\n", - "Mar float64\n", - "Apr float64\n", - "May float64\n", - "Jun float64\n", - "Jul float64\n", - "Aug float64\n", - "Sep float64\n", - "Oct float64\n", - "Nov float64\n", - "Dec float64\n", - "dtype: object\n", - "[/stdout]\n", - "inference> The csv file contains 10 rows and 13 columns. The columns are named 'Year', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'. The data types of the columns are all float64, indicating that the data is numeric. The 'Year' column is of type int64, suggesting that it contains integer values. The remaining 12 columns contain floating point numbers.\n", - "User> Plot average yearly inflation as a time series\n", - "inference> import pandas as pd\n", - "import matplotlib.pyplot as plt\n", - "\n", - "# Load data\n", - "df = pd.read_csv(\"/tmp/tmpvzjigv7g/n2OzlTWhinflation.csv\")\n", - "\n", - "# Calculate average yearly inflation\n", - "df['Average'] = df[['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']].mean(axis=1)\n", - "\n", - "# Plot average yearly inflation as a time series\n", - "plt.figure(figsize=(10,6))\n", - "plt.plot(df['Year'], df['Average'])\n", - "plt.title('Average Yearly Inflation')\n", - "plt.xlabel('Year')\n", - "plt.ylabel('Average Inflation')\n", - "plt.grid(True)\n", - "plt.show()\n", - "tool_execution> Tool:code_interpreter Args:{'code': 'import pandas as pd\\nimport matplotlib.pyplot as plt\\n\\n# Load data\\ndf = pd.read_csv(\"/tmp/tmpvzjigv7g/n2OzlTWhinflation.csv\")\\n\\n# Calculate average yearly inflation\\ndf[\\'Average\\'] = df[[\\'Jan\\', \\'Feb\\', \\'Mar\\', \\'Apr\\', \\'May\\', \\'Jun\\', \\'Jul\\', \\'Aug\\', \\'Sep\\', \\'Oct\\', \\'Nov\\', \\'Dec\\']].mean(axis=1)\\n\\n# Plot average yearly inflation as a time series\\nplt.figure(figsize=(10,6))\\nplt.plot(df[\\'Year\\'], df[\\'Average\\'])\\nplt.title(\\'Average Yearly Inflation\\')\\nplt.xlabel(\\'Year\\')\\nplt.ylabel(\\'Average Inflation\\')\\nplt.grid(True)\\nplt.show()'}\n", - "tool_execution> Tool:code_interpreter Response:completed\n", - "inference> This code calculates the average inflation for each year by taking the mean of the 12 monthly inflation rates. It then plots this average yearly inflation as a time series using matplotlib. The x-axis represents the year and the y-axis represents the average inflation. The plot shows the trend of average yearly inflation over the years.\n" - ] - } - ], - "source": [ - "from llama_stack_client.types.agents.turn_create_params import Document\n", - "\n", - "agent_config = AgentConfig(\n", - " sampling_params = {\n", - " \"max_tokens\" : 4096,\n", - " \"temperature\": 0.0\n", - " },\n", - " model=\"meta-llama/Llama-3.1-8B-Instruct\",\n", - " instructions=\"You are a helpful assistant\",\n", - " toolgroups=[\n", - " \"builtin::code_interpreter\",\n", - " \"builtin::websearch\"\n", - " ],\n", - " tool_choice=\"auto\",\n", - " input_shields=[],\n", - " output_shields=[],\n", - " enable_session_persistence=False,\n", - ")\n", - "codex_agent = Agent(client, agent_config)\n", - "session_id = codex_agent.create_session(\"test-session\")\n", - "\n", - "\n", - "inflation_doc = Document(\n", - " content=\"https://raw.githubusercontent.com/meta-llama/llama-stack-apps/main/examples/resources/inflation.csv\",\n", - " mime_type=\"text/csv\",\n", - ")\n", - "\n", - "user_input = [\n", - " {\"prompt\": \"Here is a csv, can you describe it?\", \"documents\": [inflation_doc]},\n", - " {\"prompt\": \"Plot average yearly inflation as a time series\"},\n", - "]\n", - "\n", - "for input in user_input:\n", - " cprint(f'User> {input[\"prompt\"]}', 'green')\n", - " response = codex_agent.create_turn(\n", - "\n", - " messages=[\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": input[\"prompt\"],\n", - " }\n", - " ],\n", - " session_id=session_id,\n", - " documents=input.get(\"documents\", None)\n", - " )\n", - " # for chunk in response:\n", - " # print(chunk)\n", - "\n", - " for log in EventLogger().log(response):\n", - " log.print()\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "9GHJHfLmIQQi" - }, - "source": [ - "- Now, use the generated response from agent to view the plot" - ], - "id": "9GHJHfLmIQQi" - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 564 - }, - "id": "JqBBVLKdIHHq", - "outputId": "3c89c303-e7c0-4ae2-c271-f34a4d296a85" - }, - "outputs": [ - { - "output_type": "display_data", - "data": { - "text/plain": [ - "
" - ], - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0EAAAIjCAYAAADFthA8AAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAdE5JREFUeJzt3Xd4VGX6xvF7Jpn0QhLSgBBCJwmg9CYIUqQKFlyxYF3XtZfd/bmrArquZa3r2laxgxVUQAGRJr3XQKgJoQRCEtJISJvz+yMkEgEhMMmZyXw/15VLc+ZkzjPkJcyd9z3PazEMwxAAAAAAuAmr2QUAAAAAQF0iBAEAAABwK4QgAAAAAG6FEAQAAADArRCCAAAAALgVQhAAAAAAt0IIAgAAAOBWCEEAAAAA3AohCAAAAIBbIQQBANzS5Zdfrssvv9zsMqp8+umnatu2rWw2mxo0aCCpdmqcOHGiLBaLQ58TAFwNIQgAHOytt96SxWJR9+7dzS7FaaxYsUJWq1WPP/74GR9/4YUXZLFY9MMPP9RxZY5jsVh03333XdDXJicn69Zbb1WLFi303nvv6X//+99F1VJYWKiJEydq0aJFF/U8AFBfEYIAwMGmTJmiZs2aafXq1dq9e7fZ5TiFnj176u6779bLL7+spKSkao/t27dPTz/9tK677joNHz7cpArNtWjRItntdr3++uu69dZbNXbs2It6vsLCQk2aNOmMIeiJJ55QUVHRRT0/ALg6QhAAOFBKSoqWL1+uV155ReHh4ZoyZUqd12C323XixIk6v+65PP/882rYsKHuvvtuGYZRdfz++++XzWbT66+/Xid1FBYW1sl1aiIjI0OSqpbB1SZPT0/5+PjU+nUAwJkRggDAgaZMmaKQkBANHz5c1157bbUQVFpaqtDQUN12222nfV1eXp58fHz02GOPVR0rLi7WhAkT1LJlS3l7eysmJkZ//etfVVxcXO1rK5dhTZkyRQkJCfL29tacOXMkSS+99JJ69eqlsLAw+fr6qnPnzvrmm29Ou35RUZEeeOABNWzYUIGBgRo1apQOHjwoi8WiiRMnVjv34MGDuv322xUZGSlvb28lJCTogw8+OOefTXBwsF5//XUtW7ZM77//viTp22+/1cyZM/X8888rOjpadrtdr732mhISEuTj46PIyEjdfffdOnbsWLXn+v777zV8+HA1atRI3t7eatGihZ555hmVl5dXO+/yyy9XYmKi1q1bp759+8rPz09///vfT6utoKBA/v7+evDBB0977MCBA/Lw8NBzzz13ztd4qkWLFsliseirr77Ss88+qyZNmsjHx0dXXHFFtRnCZs2aacKECZKk8PDwM/6ZVyopKdFTTz2lzp07Kzg4WP7+/rrsssu0cOHCqnNSU1MVHh4uSZo0aZIsFku15zzTPUFlZWV65pln1KJFC3l7e6tZs2b6+9//ftpYa9asmUaMGKGlS5eqW7du8vHxUfPmzfXJJ5/U6M8GAExnAAAcpm3btsYdd9xhGIZh/PLLL4YkY/Xq1VWP33777UaDBg2M4uLial/38ccfG5KMNWvWGIZhGOXl5cbgwYMNPz8/46GHHjLeffdd47777jM8PT2Nq666qtrXSjLatWtnhIeHG5MmTTLefPNNY8OGDYZhGEaTJk2MP//5z8Z///tf45VXXjG6detmSDJmzZpV7TnGjh1rSDJuvvlm48033zTGjh1rdOzY0ZBkTJgwoeq8w4cPG02aNDFiYmKMp59+2nj77beNUaNGGZKMV1999bz+jIYPH26EhIQYe/bsMWJiYoxevXoZdrvdMAzDuPPOOw1PT0/jrrvuMt555x3jb3/7m+Hv72907drVKCkpqXqO0aNHG2PHjjX+/e9/G2+//bZx3XXXGZKMxx57rNq1+vXrZ0RFRRnh4eHG/fffb7z77rvGd999V/VYv379qs698cYbjcjISKOsrKzac7z44ouGxWIx9u3b97uvS5Jx7733Vn2+cOFCQ5Jx6aWXGp07dzZeffVVY+LEiYafn5/RrVu3qvO+/fZbY8yYMYYk4+233zY+/fRTY9OmTWes8ejRo0Z0dLTxyCOPGG+//bbx4osvGm3atDFsNlvV97ygoMB4++23DUnGmDFjjE8//bTac06YMMH47T//48ePNyQZ1157rfHmm28at9xyiyHJGD16dLXzYmNjjTZt2hiRkZHG3//+d+O///2v0alTJ8NisRhbt2793T8fAHAmhCAAcJC1a9cakox58+YZhmEYdrvdaNKkifHggw9WnTN37lxDkjFz5sxqXzts2DCjefPmVZ9/+umnhtVqNZYsWVLtvHfeeceQZCxbtqzqmCTDarUaSUlJp9VUWFhY7fOSkhIjMTHRGDBgQNWxdevWGZKMhx56qNq5t95662kh6I477jCio6ONzMzMauf+4Q9/MIKDg0+73pmkpqYa/v7+RmhoqGGz2YwtW7YYhmEYS5YsMSQZU6ZMqXb+nDlzTjt+puvcfffdhp+fn3HixImqY/369TMkGe+8885p5/82YFR+b2bPnl3tvA4dOlQ772zOFoLatWtXLfS+/vrrhqSq120YvwaTo0eP/m6NZWVlpwXoY8eOGZGRkcbtt99edezo0aOnfe9+e61KGzduNCQZd955Z7XzHnvsMUOSsWDBgqpjsbGxhiTjl19+qTqWkZFheHt7G48++ujZ/mgAwOmwHA4AHGTKlCmKjIxU//79JVUsU7v++uv1xRdfVC3TGjBggBo2bKgvv/yy6uuOHTumefPm6frrr6869vXXX6tdu3Zq27atMjMzqz4GDBggSdWWP0lSv379FB8ff1pNvr6+1a6Tm5uryy67TOvXr686Xrl07s9//nO1r73//vurfW4YhqZNm6aRI0fKMIxqdQ0ZMkS5ubnVnvdsYmNjNWHCBGVnZ+uRRx5RYmJi1WsODg7WoEGDqj13586dFRAQUO01n/q68vPzlZmZqcsuu0yFhYVKTk6udj1vb+8zLkH8rYEDB6pRo0bVljBu3bpVmzdv1k033XTOrz+b2267TV5eXlWfX3bZZZKkvXv31vi5PDw8qp7LbrcrOztbZWVl6tKly3n92Z/Jjz/+KEl65JFHqh1/9NFHJem0jn3x8fFVr0GqWMLXpk2bC3o9AGAWT7MLAID6oLy8XF988YX69++vlJSUquPdu3fXyy+/rPnz52vw4MHy9PTUNddco6lTp6q4uFje3t6aPn26SktLq4WgXbt2afv27VX3dvxW5Y30leLi4s543qxZs/TPf/5TGzdurHZ/x6n3hOzbt09Wq/W052jZsmW1z48ePaqcnBz973//O2sL59/WdTZdu3aVJHXp0qXq2K5du5Sbm6uIiIhzPndSUpKeeOIJLViwQHl5edXOy83NrfZ548aNq4WQs7Farbrxxhv19ttvq7CwUH5+fpoyZYp8fHx03XXXndfrOpOmTZtW+zwkJESSTrvP6Xx9/PHHevnll5WcnKzS0tKq42cbA+dS+f3/7fc7KipKDRo00L59+6od/+3rkSpe04W+HgAwAyEIABxgwYIFSk9P1xdffKEvvvjitMenTJmiwYMHS5L+8Ic/6N1339Xs2bM1evRoffXVV2rbtq06duxYdb7dblf79u31yiuvnPF6MTEx1T4/dWak0pIlSzRq1Cj17dtXb731lqKjo2Wz2fThhx9q6tSpNX6NdrtdknTTTTdp/PjxZzynQ4cONX7eU58/IiLirB31KgNhTk6O+vXrp6CgID399NNq0aKFfHx8tH79ev3tb3+rqrPSmf5szuaWW27Rv//9b3333Xe64YYbNHXqVI0YMULBwcEX/Lo8PDzOeNw4pUPe+frss8906623avTo0frLX/6iiIiIqqYNe/bsueAaJZ33BqqOfD0AYBZCEAA4wJQpUxQREaE333zztMemT5+ub7/9Vu+88458fX3Vt29fRUdH68svv1SfPn20YMEC/eMf/6j2NS1atNCmTZt0xRVXnPeb09+aNm2afHx8NHfuXHl7e1cd//DDD6udFxsbK7vdrpSUFLVq1arq+G/3OAoPD1dgYKDKy8s1cODAC6rp97Ro0UI///yzevfu/bvBZdGiRcrKytL06dPVt2/fquOnzsBdqMTERF166aWaMmWKmjRporS0NL3xxhsX/byO8s0336h58+aaPn16tXFR2V2uUk3GTOX3f9euXWrXrl3V8SNHjignJ0exsbEXXzgAOBnuCQKAi1RUVKTp06drxIgRuvbaa0/7uO+++5Sfn68ZM2ZIqlh2de2112rmzJn69NNPVVZWVm0pnCSNHTtWBw8e1HvvvXfG6x0/fvycdXl4eMhisVRrG52amqrvvvuu2nlDhgyRJL311lvVjv/2zb+Hh4euueYaTZs2TVu3bj3tekePHj1nTb9n7NixKi8v1zPPPHPaY2VlZcrJyamqQ6o+81BSUnJa/Rfq5ptv1k8//aTXXntNYWFhGjp0qEOe1xHO9NpXrVqlFStWVDvPz89Pkqr+zH7PsGHDJEmvvfZateOVs5DuuoEtgPqNmSAAuEgzZsxQfn6+Ro0adcbHe/ToUbVxamXYuf766/XGG29owoQJat++fbXfwEsVb8S/+uor/elPf9LChQvVu3dvlZeXKzk5WV999ZXmzp1b7X6aMxk+fLheeeUVXXnllRo3bpwyMjL05ptvqmXLltq8eXPVeZ07d9Y111yj1157TVlZWerRo4cWL16snTt3Sqo+q/D8889r4cKF6t69u+666y7Fx8crOztb69ev188//6zs7OwL+jOUKpo73H333Xruuee0ceNGDR48WDabTbt27dLXX3+t119/Xddee6169eqlkJAQjR8/Xg888IAsFos+/fRThy3HGjdunP7617/q22+/1T333CObzeaQ53WEESNGaPr06RozZoyGDx+ulJQUvfPOO4qPj1dBQUHVeb6+voqPj9eXX36p1q1bKzQ0VImJiVVNKE7VsWNHjR8/Xv/73/+qlhquXr1aH3/8sUaPHl3V6AMA6hNCEABcpMqb5wcNGnTGx61Wq4YPH64pU6YoKytLYWFh6tWrl2JiYrR///7TZoEqv+a7777Tq6++qk8++UTffvut/Pz81Lx5cz344INq3br1OesaMGCAJk+erOeff14PPfSQ4uLi9MILLyg1NbVaCJKkTz75RFFRUfr888/17bffauDAgfryyy/Vpk0b+fj4VJ0XGRmp1atX6+mnn9b06dP11ltvKSwsTAkJCXrhhRdq+Cd3unfeeUedO3fWu+++q7///e/y9PRUs2bNdNNNN6l3796SpLCwMM2aNUuPPvqonnjiCYWEhOimm27SFVdcUTWrdTEiIyM1ePBg/fjjj7r55psv+vkc6dZbb9Xhw4f17rvvau7cuYqPj9dnn32mr7/+WosWLap27vvvv6/7779fDz/8sEpKSjRhwoQzhqDKc5s3b66PPvpI3377raKiovT444+ftswOAOoLi8GdjACAM9i4caMuvfRSffbZZ7rxxhvNLqdOjRkzRlu2bDntvigAQP3APUEAABUVFZ127LXXXpPVaq3WfMAdpKen64cffnC6WSAAgOOwHA4AoBdffFHr1q1T//795enpqdmzZ2v27Nn64x//eFo77voqJSVFy5Yt0/vvvy+bzaa7777b7JIAALWEEAQAUK9evTRv3jw988wzKigoUNOmTTVx4sTTWnfXZ4sXL9Ztt92mpk2b6uOPP1ZUVJTZJQEAagn3BAEAAABwK9wTBAAAAMCtEIIAAAAAuBWXvifIbrfr0KFDCgwMrLaZHwAAAAD3YhiG8vPz1ahRI1mtvz/X49Ih6NChQ27TtQgAAADAue3fv19NmjT53XNcOgQFBgZKqnihQUFBptZSWlqqn376SYMHD5bNZjO1FrgHxhzqGmMOdYnxhrrGmHN9eXl5iomJqcoIv8elQ1DlErigoCCnCEF+fn4KCgriLw7qBGMOdY0xh7rEeENdY8zVH+dzmwyNEQAAAAC4FUIQAAAAALdCCAIAAADgVghBAAAAANwKIQgAAACAWyEEAQAAAHArhCAAAAAAboUQBAAAAMCtEIIAAAAAuBVCEAAAAAC3QggCAAAA4FYIQQAAAADcCiEIAAAAgFshBAEAAABwK4QgAAAAAG6FEAQAAADArRCCAAAA4NYMw9CGtByVlJtdCeoKIQgAAABubebmdI19b7U+2GmVYRhml4M6QAgCAACAW/tuw0FJ0vYcq+YkHTG5GtQFQhAAAADcVv6JUi3dlVn1+b9m79Dx4jITK0JdIAQBAADAbS1IzlBJuV2xoX4K8zZ0OK9Y/1mwy+yyUMsIQQAAAHBbc7YeliQNS4zU1XF2SdLkJSnanZFvZlmoZYQgAAAAuKWiknIt2nFUkjQ4PlKJIYauaBuuMruhJ79LoklCPUYIAgAAgFtavPOoikrL1biBrxIaBUqS/jGsjbw9rVqxN0szN6ebXCFqCyEIAAAAbmluUsVSuCsTo2SxWCRJMSF+urd/S0nSP2dtU/6JUtPqQ+0hBAEAAMDtlJTZ9fP2inbYQxOjqj32x77N1SzMTxn5xXr9Z5ok1EeEIAAAALid5XsylX+iTOGB3urUNKTaYz42D00clSBJ+nB5qnYcpklCfUMIAgAAgNupXAo3JCFSVqvltMcvbxOhKxOiVG439OT3W2mSUM+YHoIOHjyom266SWFhYfL19VX79u21du1as8sCAABAPVVuN/RTUsVSuCsTos963pMj4+Vr89DqlGx9t/FgXZWHOmBqCDp27Jh69+4tm82m2bNna9u2bXr55ZcVEhJy7i8GAAAALsCa1GxlHS9RAz+bujcPPet5jRv46v4rKpokPPtDsnKLaJJQX3iaefEXXnhBMTEx+vDDD6uOxcXFmVgRAAAA6rvKDVIHtouUzeP35wTu7NNc36w7oL1Hj+vVeTur7hWCazM1BM2YMUNDhgzRddddp8WLF6tx48b685//rLvuuuuM5xcXF6u4uLjq87y8PElSaWmpSkvNTeaV1ze7DrgPxhzqGmMOdYnxhtpitxuavbVi/59B7cJPG2u/HXMWSU8Nb6tbP1qnT1ak6upLotUuOrBOa8b5qcnPC4th4l1ePj4+kqRHHnlE1113ndasWaMHH3xQ77zzjsaPH3/a+RMnTtSkSZNOOz516lT5+fnVer0AAABwban50qtbPeVtNfRs13LZzvPmkI92WrUhy6q4QEMPJJTrDL0UYLLCwkKNGzdOubm5CgoK+t1zTQ1BXl5e6tKli5YvX1517IEHHtCaNWu0YsWK084/00xQTEyMMjMzz/lCa1tpaanmzZunQYMGyWazmVoL3ANjDnWNMYe6xHhDbXlx7k69tzRVwxOj9Nr1HaqOn2vMpeee0JX/WabCknI9PyZB13RqXJdl4zzk5eWpYcOG5xWCTF0OFx0drfj4+GrH2rVrp2nTpp3xfG9vb3l7e5923GazOc0PSGeqBe6BMYe6xphDXWK8wZEMw9BP2zMkScM6NDrj2DrbmGva0KaHBrbSv35M1r9/2qWh7Rsr2I+x6Uxq8rPC1O5wvXv31o4dO6od27lzp2JjY02qCAAAAPVV8uF87csqlLenVZe3Ca/x19/WO06tIgKUdbxEL/2049xfAKdlagh6+OGHtXLlSv3rX//S7t27NXXqVP3vf//Tvffea2ZZAAAAqIdmn+wK17d1uPy9a74gyuZh1dNXJUqSPlu1T1sO5Dq0PtQdU0NQ165d9e233+rzzz9XYmKinnnmGb322mu68cYbzSwLAAAA9dDckyHoyoSoC36Oni3CdNUljWQY0hPfb5Xdbtrt9bgIpt4TJEkjRozQiBEjzC4DAAAA9djeowXacSRfnlaLBraLvKjn+sewdpq/PUOb9ufoq7X79YduTR1UJeqKqTNBAAAAQF2Yk1QxC9SzRdhFNzSICPLRw4NaS5JemJOsY8dLLro+1C1CEAAAAOq9yqVwQxOjHfJ843vGqm1UoI4VlurFuTRJcDWEIAAAANRrB3OKtOlAriwWaVD8xS2Fq+R5SpOEL9akaeP+HIc8L+oGIQgAAAD1WuUsUNdmoQoPPH3PyQvVLS5UV3dqLMOQnvxuq8ppkuAyCEEAAACo1+Y4oCvc2Tw+tJ0CfTy15WCupq5Oc/jzo3YQggAAAFBvHc0v1pp92ZKkKxMdH4LCA7312OA2kqR/z0lWVkGxw68BxyMEAQAAoN76adthGYbUsUmwGjXwrZVr3Ni9qeKjg5R3okwvzEmulWvAsQhBAAAAqLcql8INqYVZoEqeHlY9M7qiScJXaw9o3cmZJzgvQhAAAADqpdzCUq3YkyWpdu4HOlXn2BCN7dJEkvTkd0kqK7fX6vVwcQhBAAAAqJd+3n5EZXZDbSID1Tw8oNav97cr2yrY16Zt6Xn6bOW+Wr8eLhwhCAAAAPXSnKSTXeFqcSncqcICvPWXIRVNEl7+aaeO5tMkwVkRggAAAFDvHC8u0y87j0qquxAkSTd0a6oOTYKVX1ym52Zvr7PromYIQQAAAKh3Fu04quIyu5qF+altVGCdXdfDatEzVyXKYpGmrz+o1Sk0SXBGhCAAAADUO7O3pkuq6ApnsVjq9NodYxroD12bSpKe/G6rSmmS4HQIQQAAAKhXTpSWa2FyhiRpaGK0KTX8dUgbhfjZtONIvj5enmpKDTg7QhAAAADqlaW7MnW8pFzRwT7q0DjYlBpC/L30tyvbSpJe+3mXjuSdMKUOnBkhCAAAAPVKZVe4IQlRslrrdincqcZ2idElMQ1UUFymZ3+gSYIzIQQBAACg3igtt2vetiOS6rYr3JlYrRb9c3RFk4QZmw5p+Z5MU+vBrwhBAAAAqDdW7c1WblGpwvy91LVZqNnlKLFxsG7qHitJeur7JJokOAlCEAAAAOqNyq5wgxMi5WHiUrhTPTa4jcL8vbQ7o0AfLE0xuxyIEAQAAIB6otxuaG5SxVK4IQnmLoU7VbCfTf83tKJJwuvzdyk9t8jkikAIAgAAQL2wPu2YMguKFejjqV4tGppdTjXXdGqiLrEhKiwp1z9n0STBbIQgAAAA1AtztlZ0hRvYLlJens71NtdqtejpqxJltUg/bEnXkl1HzS7JrTnX6AAAAAAugGEYVSHI7K5wZxPfKEi39GwmSZrwfZKKy8rNLciNEYIAAADg8rYezNPBnCL52jzUt1W42eWc1SODW6thgLf2Zh7X+0tokmAWQhAAAABc3pykiq5w/duGy9fLw+Rqzi7Ix6Z/DK9okvDGgl06mEOTBDMQggAAAODSDMPQ7JNL4ZypK9zZjL6ksbrFhepEqV1Pz0wyuxy3RAgCAACAS9udUaC9R4/Ly8OqAW0jzC7nnCwWi565KlEeVovmJh3Rwh0ZZpfkdghBAAAAcGmVs0B9WjVUoI/N5GrOT5uoQN3Wq5kkaeKMJJ0opUlCXSIEAQAAwKU5e1e4s3loUGtFBnlrX1ah/vfLXrPLcSuEIAAAAListKxCbUvPk4fVooHtIs0up0YCvD31j+HxkqQ3F+7W/uxCkytyH4QgAAAAuKzKrnDd40IV6u9lcjU1N7JDtHq1CFNxmV2TaJJQZwhBAAAAcFmV9wMNdbGlcJUsFouevipBNg+Lft6eoZ+3HTG7JLdACAIAAIBLOpx7QhvSciRJg12gNfbZtIwI1B19mkuSJs2iSUJdIAQBAADAJc1NqpgF6hwbosggH5OruTj3D2ip6GAf7c8u0luL9phdTr1HCAIAAIBLquoK58KzQJX8vT315IiKJgnvLN6j1MzjJldUvxGCAAAA4HKyj5doVUqWJNdrjX02QxOjdFmrhiops2vizCQZhmF2SfUWIQgAAAAuZ962w7IbUkKjIMWE+pldjkNYLBZNGpUgLw+rFu04qp9oklBrCEEAAABwOXNcvCvc2TQPD9Af+1Y0SXh65jYVlpSZXFH9RAgCAACAS8k7UaqluzMl1Z+lcKe6t39LNW7gq4M5RXpz4W6zy6mXCEEAAABwKQuTM1RabqhlRIBaRgSaXY7D+Xp5aMLIiiYJ//tlr/YcLTC5ovqHEAQAAACXMntL/ekKdzaD4iPVv024SssNTZxBkwRHIwQBAADAZRSVlGvRzgxJ9XMpXCWLxaKJoxLk5WnVkl2Z+vFk8INjEIIAAADgMhbvzNCJUruahPgqoVGQ2eXUqtgwf93Tr4Uk6ZlZ23S8mCYJjkIIAgAAgMs4dYNUi8VicjW1757LWygm1FeH807oPwt2mV1OvUEIAgAAgEsoLivX/O0VS+GGtq+/S+FO5WPz0KRRCZKkyUtStOtIvskV1Q+EIAAAALiE5XuylF9cpohAb10aE2J2OXVmQNtIDWwXqTK7oae+p0mCIxCCAAAA4BLmnlwKNyQhSlZr/V8Kd6oJI+Pl7WnVir1ZmrHpkNnluDxCEAAAAJxeWbldP207Iql+d4U7m5hQP93Xv6Uk6dkftiv/RKnJFbk2QhAAAACc3prUY8o+XqIGfjZ1jws1uxxT3NW3uZqF+Skjv1iv/UyThItBCAIAAIDTm7M1XZI0qF2kPD3c8y2sj81DE082SfhoeaqSD+eZXJHrcs8RBAAAAJdhtxuam1SxFM5dusKdzeVtInRlQpTK7Yae+o4mCReKEAQAAACntvFAjg7nnVCAt6d6t2xodjmme3JkvHxtHlqdmq1vNxw0uxyXRAgCAACAU6vsCjegbYS8PT1MrsZ8jRv46v4rKpok/OvH7cotoklCTRGCAAAA4LQMw9DskyHIHbvCnc2dfZqrebi/MgtK9Oq8nWaX43IIQQAAAHBa29PzlZZdKG9Pqy5vE252OU7Dy9Oqp0clSpI+WZGqpEO5JlfkWghBAAAAcFqVXeH6tQ6Xn5enydU4lz6tGmp4h2jZDemp75Nkt9Mk4XwRggAAAOC05iSxFO73PDk8Xn5eHlq375i+WX/A7HJcBiEIAAAATmnP0QLtPFIgT6tFV7SLNLscpxQV7KOHBraSJD0/O1m5hTRJOB+EIAAAADilOScbIvRq2VDBvjaTq3Fet/WOU6uIAGUfL9G/f0o2uxyXQAgCAACAU5p7cincUJbC/S6bh1VPX1XRJGHKqjRtOUCThHMhBAEAAMDpHDhWqM0HcmW1SIPiWQp3Lj1bhOmqSxrJMKQnvt9Kk4RzIAQBAADA6cxNOiJJ6tosVA0DvE2uxjX8Y1g7BXh7atP+HH25dr/Z5Tg1QhAAAACcTmVrbLrCnb+IIB89PKi1JOmFOck6drzE5IqcFyEIAAAATiUj/4TW7jsmSRqSQAiqifE9Y9U2KlA5haV6cS5NEs6GEAQAAACn8lPSERmG1DGmgRo18DW7HJfieUqThC/W7NeGtGMmV+ScCEEAAABwKnSFuzjd4kJ1dafGMgzpye+3qpwmCachBAEAAMBp5BSWaMWeLEnSlSyFu2CPD22nQB9PbT2Yp6mr08wux+kQggAAAOA0ft6eoTK7obZRgWrW0N/sclxWeKC3HhvcRpL07znJyiwoNrki50IIAgAAgNOgK5zj3NQjVgmNgpR3okwvzKZJwqkIQQAAAHAKBcVl+mVXpiRCkCN4WC1VTRK+XndA6/Zlm1yR8yAEAQAAwCks2pGhkjK74hr6q01koNnl1AudY0N0fZcYSdIT3yWprNxuckXOgRAEAAAApzB7a0VXuCEJUbJYLCZXU3/89co2Cva1aXt6nj5buc/scpwCIQgAAACmO1FaroXJGZJoje1oYQHe+suQiiYJL/+0Uxn5J0yuyHyEIAAAAJhuya5MFZaUq1Gwjzo0CTa7nHrnhm5N1aFJsPKLy/T8jzRJIAQBAADAdHMql8IlshSuNnhYLXrmqkRZLNL0DQe1am+W2SWZihAEAAAAU5WW2/Xz9iOS2CC1NnWMaaAbujWVJD31fZJK3bhJAiEIAAAAplq5N0u5RaVqGOClLs1CzS6nXvvL4DYK8bNpx5F8fbw81exyTEMIAgAAgKkqu8INio+Sh5WlcLUpxN9L/ze0rSTptZ936UieezZJIAQBAADANOV2Qz8lVSyFoytc3biuc4wuiWmgguIyPfvDdrPLMQUhCAAAAKZZt++YMguKFeTjqR7Nw8wuxy1YrRb9c3SirBZpxqZDWr470+yS6hwhCAAAAKap7Ao3sF2kvDx5a1pXEhsH66YesZKkp2YkqaTMvZokMNIAAABgCsMwNDepIgRdyVK4OvfooDYK8/fS7owCfbgsxexy6pSpIWjixImyWCzVPtq2bWtmSQAAAKgjWw7m6mBOkfy8PNS3dbjZ5bidYD+bHh/WTpL0+vxdSs8tMrmiumP6TFBCQoLS09OrPpYuXWp2SQAAAKgDlUvh+reJkI/Nw+Rq3NPVlzZWl9gQFZaU65+z3KdJgukhyNPTU1FRUVUfDRs2NLskAAAA1DLDMKpC0BCWwpnGarXo6asqmiT8sCVdv+w8anZJdcLT7AJ27dqlRo0aycfHRz179tRzzz2npk2bnvHc4uJiFRcXV32el5cnSSotLVVpaWmd1Hs2ldc3uw64D8Yc6hpjDnWJ8Vb/7TpSoL2Zx2XzsOiyFiGmf6/decy1CvfVzT2a6uMVaZrw/VbNvK+XvF2wSUVNvncWwzCMWqzld82ePVsFBQVq06aN0tPTNWnSJB08eFBbt25VYGDgaedPnDhRkyZNOu341KlT5efnVxclAwAAwAHm7Ldo9gEPJYTY9ce27tWZzBkVlUn/2uihvFKLhseUa3AT0yLCBSssLNS4ceOUm5uroKCg3z3X1BD0Wzk5OYqNjdUrr7yiO+6447THzzQTFBMTo8zMzHO+0NpWWlqqefPmadCgQbLZbKbWAvfAmENdY8yhLjHe6r+Rb65Q8uF8PT8mQdd0amx2OYw5Sd9vStdj32yRj82qOQ/0VuMGvmaXVCN5eXlq2LDheYUg05fDnapBgwZq3bq1du/efcbHvb295e3tfdpxm83mNIPVmWqBe2DMoa4x5lCXGG/1076s40o+nC8Pq0VDEhs51ffYncfcNZ1j9PW6g1qVkq3n5uzUuzd3MbukGqnJ982pFvsVFBRoz549io6ONrsUAAAA1JLKhgg9m4cpxN/L5GpQyWKx6JnRifKwWjQ36YgW7sgwu6RaY2oIeuyxx7R48WKlpqZq+fLlGjNmjDw8PHTDDTeYWRYAAABq0Wy6wjmt1pGBur13M0nSxBlJOlFabm5BtcTUEHTgwAHdcMMNatOmjcaOHauwsDCtXLlS4eFslgUAAFAfpecWaeP+HFks0pD4SLPLwRk8OLC1IoO8tS+rUO8u3mt2ObXC1HuCvvjiCzMvDwAAgDo29+QsUOemIYoI8jG5GpxJgLennhger/s/36C3Fu3WmEsbq2lY/erE7FT3BAEAAKB+m5NUEYKuZCmcUxvRIVq9WoSpuMyuSTOTzC7H4QhBAAAAqBNZBcVanZItSRqSQAhyZhaLRU9flSibh0XzkzP087YjZpfkUIQgAAAA1Il5247IbkiJjYMUE1q/llfVRy0jAnRHn+aSpIkz61eTBEIQAAAA6kTlUrihiWyH4iruH9BS0cE+OnCsSG8tPPNenq6IEAQAAIBal1tUqmW7MyWxFM6V+Ht76qkR8ZKkdxbvVWrmcZMrcgxCEAAAAGrdwuQMlZYbahURoJYRAWaXgxq4MjFKl7VqqJJyuybMSJJhGGaXdNEIQQAAAKh1s7emS6IrnCuqbJLg5WHV4p1HNTfJ9ZskEIIAAABQqwpLyrR451FJhCBXFdfQX3/sW9Ek4ZlZ21RYUmZyRReHEAQAAIBatXjHUZ0otSsm1Ffx0UFml4MLdG//lmrcwFcHc4r03wWu3SSBEAQAAIBadWpXOIvFYnI1uFC+Xh6aMLKiScJ7S/Zqz9ECkyu6cIQgAAAA1JrisnIt2J4hia5w9cGg+Ej1bxOu0nJDE7533SYJhCAAAADUmuW7s5RfXKbIIG9dGtPA7HJwkSwWiyaOSpCXp1VLd2fqxy2HzS7pghCCAAAAUGsqu8INSYiS1cpSuPogNsxf9/RrIamiSUJBses1SSAEAQAAoFaUlds1b1tFO+UrWQpXr9xzeQs1DfXT4bwTemP+LrPLqTFCEAAAAGrF6tRsHSssVYifTd3iQs0uBw7kY/PQxFEVTRImL01RauZxkyuqGU+zCwAAAED9NGdrxf0ig+Ij5enB797rmwFtI3VDtxh1bRaq2DA/s8upEUIQAAAAHM5uNzT3lNbYqJ+eu7qD2SVcECI5AAAAHG7D/hwdyStWoLenerUMM7scoBpCEAAAAByuchZoQLsIeXt6mFwNUB0hCAAAAA5lGEZVa2y6wsEZEYIAAADgUNvS87Q/u0g+Nqv6tQk3uxzgNIQgAAAAOFRlV7h+rcPl50UfLjgfQhAAAAAcqjIE0RUOzooQBAAAAIfZnVGgXRkFsnlY1L9thNnlAGdECAIAAIDDVHaF692yoYJ9bSZXA5wZIQgAAAAOQ1c4uAJCEAAAABxif3ahth7Mk9UiDYqPNLsc4KwIQQAAAHCIyqVw3eJCFRbgbXI1wNkRggAAAOAQlV3hWAoHZ0cIAgAAwEXLyDuhdWnHJElDEglBcG6EIAAAAFy0uduOyDCkS2IaKDrY1+xygN9FCAIAAMBFm1u1QSqzQHB+hCAAAABclGPHS7Rib5Yk6UpCEFwAIQgAAAAX5eftR1RuN9QuOkixYf5mlwOcEyEIAAAAF4WucHA1hCAAAABcsILiMi3ZlSlJGtqeEATXQAgCAADABVuQnKGScruaN/RXq4gAs8sBzgshCAAAABessivckMQoWSwWk6sBzo/nhXxRTk6OVq9erYyMDNnt9mqP3XLLLQ4pDAAAAM7tRGm5Fu7IkERrbLiWGoegmTNn6sYbb1RBQYGCgoKqJX6LxUIIAgAAcBO/7DyqwpJyNW7gq/aNg80uBzhvNV4O9+ijj+r2229XQUGBcnJydOzYsaqP7Ozs2qgRAAAATmhO0smlcAkshYNrqXEIOnjwoB544AH5+fnVRj0AAABwASVldv287YgkNkiF66lxCBoyZIjWrl1bG7UAAADARazcm6W8E2VqGOCtzrEhZpcD1EiN7wkaPny4/vKXv2jbtm1q3769bDZbtcdHjRrlsOIAAADgnGaf7Ao3OCFSHlaWwsG11DgE3XXXXZKkp59++rTHLBaLysvLL74qAAAAOK1yu6F52ypCEF3h4IpqHIJ+2xIbAAAA7mVtarYyC0oU7GtTj+ZhZpcD1BibpQIAAKBGKrvCDWwXKZsHbyfhei5o1C5evFgjR45Uy5Yt1bJlS40aNUpLlixxdG0AAABwMoZhaO7J+4HoCgdXVeMQ9Nlnn2ngwIHy8/PTAw88oAceeEC+vr664oorNHXq1NqoEQAAAE5i84FcHco9IT8vD13WqqHZ5QAXpMb3BD377LN68cUX9fDDD1cde+CBB/TKK6/omWee0bhx4xxaIAAAAJxHZVe4/m0j5GPzMLka4MLUeCZo7969Gjly5GnHR40apZSUFIcUBQAAAOdjGIbmbE2XJF2ZwFI4uK4ah6CYmBjNnz//tOM///yzYmJiHFIUAAAAnM/OIwVKzSqUl6dV/dtGmF0OcMFqvBzu0Ucf1QMPPKCNGzeqV69ekqRly5bpo48+0uuvv+7wAgEAAOAcZp+cBerbqqECvGv8NhJwGjUevffcc4+ioqL08ssv66uvvpIktWvXTl9++aWuuuoqhxcIAAAA5zCnqitctMmVABfngiL8mDFjNGbMGEfXAgAAACeVmnlcyYfz5Wm1aGA7lsLBtbG7FQAAAM6pcoPUni3C1MDPy+RqgItzXjNBoaGh2rlzpxo2bKiQkBBZLJaznpudne2w4gAAAOAcKltjD6ErHOqB8wpBr776qgIDA6v+//dCEAAAAOqXQzlF2rQ/RxaLNDgh0uxygIt2XiFo/PjxVf9/66231lYtAAAAcEJzTy6F6xIboohAH5OrAS5eje8J8vDwUEZGxmnHs7Ky5OHBrsEAAAD1DV3hUN/UOAQZhnHG48XFxfLy4iY5AACA+iSzoFhrUivu+R7CUjjUE+fdIvs///mPJMlisej9999XQEBA1WPl5eX65Zdf1LZtW8dXCAAAANPM23ZEdkPq0CRYTUL8zC4HcIjzDkGvvvqqpIqZoHfeeafa0jcvLy81a9ZM77zzjuMrBAAAgGnoCof66LxDUEpKiiSpf//+mj59ukJCQmqtKAAAAJgvt6hUy3dnSpKuTCQEof447xBUaeHChbVRBwAAAJzMguQjKrMbah0ZoBbhAef+AsBF1DgESdKBAwc0Y8YMpaWlqaSkpNpjr7zyikMKAwAAgLlmbznZFY6lcKhnahyC5s+fr1GjRql58+ZKTk5WYmKiUlNTZRiGOnXqVBs1AgAAoI4VlpRp8c6jkmiNjfqnxi2yH3/8cT322GPasmWLfHx8NG3aNO3fv1/9+vXTddddVxs1AgAAoI4t2nFUxWV2NQ31U7voQLPLARyqxiFo+/btuuWWWyRJnp6eKioqUkBAgJ5++mm98MILDi8QAAAAda9yg9ShiVGyWCwmVwM4Vo1DkL+/f9V9QNHR0dqzZ0/VY5mZmY6rDAAAAKYoLivXguQMSdIQusKhHqrxPUE9evTQ0qVL1a5dOw0bNkyPPvqotmzZounTp6tHjx61USMAAADq0LLdmSooLlNUkI8uadLA7HIAh6txCHrllVdUUFAgSZo0aZIKCgr05ZdfqlWrVnSGAwAAqAcqu8INSYiU1cpSONQ/NQ5BzZs3r/p/f39/vfPOOw4tCAAAAOYpK7dr3vYjkugKh/qrxvcEAQAAoP5alZKtnMJShfp7qWuzELPLAWrFec0EhYSEnHdXkOzs7IsqCAAAAOap7Ao3OD5Snh78vhz103mFoNdee62WywAAAIDZ7HZDc5NO3g9EVzjUY+cVgjZt2qRnnnlG/v7++uWXX9SrVy95etb4diIAAAA4sQ37jykjv1iB3p7q1SLM7HKAWnNec5xvvPFGVUe4/v37s+QNAACgHqpcCndFuwh5e3qYXA1Qe85rOqdZs2b6z3/+o8GDB8swDK1YsUIhIWe+Ua5v374OLRAAAAC1zzAMzT4Zgq5kKRzqufMKQf/+97/1pz/9Sc8995wsFovGjBlzxvMsFovKy8sdWiAAAABqX9KhPB04ViQfm1X9WkeYXQ5Qq84rBI0ePVqjR49WQUGBgoKCtGPHDkVE8JcDAACgvqhcCnd56wj5erEUDvVbjbobBAQEaOHChYqLi6MxAgAAQD0y52RXuKHtWQqH+q/GSaZfv36y2+3auXOnMjIyZLfbqz3OPUEAAACuZXdGvnZnFMjmYVH/tqz2Qf1X4xC0cuVKjRs3Tvv27ZNhGNUe454gAAAA11O5FK5Py4YK8rGZXA1Q+2q8DfCf/vQndenSRVu3blV2draOHTtW9XExrbOff/55WSwWPfTQQxf8HAAAAKg5usLB3dR4JmjXrl365ptv1LJlS4cVsWbNGr377rvq0KGDw54TAAAA57Y/u1BJh/JktUiD4glBcA81ngnq3r27du/e7bACCgoKdOONN+q99947695DAAAAqB2VS+G6x4Up1N/L5GqAulHjmaD7779fjz76qA4fPqz27dvLZqu+brSmszn33nuvhg8froEDB+qf//zn755bXFys4uLiqs/z8vIkSaWlpSotLa3RdR2t8vpm1wH3wZhDXWPMoS4x3urO7K3pkqTB8eFu/efNmHN9NfneWYzfdjc4B6v19Mkji8UiwzBq3Bjhiy++0LPPPqs1a9bIx8dHl19+uS655BK99tprZzx/4sSJmjRp0mnHp06dKj8/v/O+LgAAAKTcEumpdRW/E5/UqUwNvE0uCLgIhYWFGjdunHJzcxUUFPS759Z4JiglJeWCCzvV/v379eCDD2revHny8fE5r695/PHH9cgjj1R9npeXp5iYGA0ePPicL7S2lZaWat68eRo0aNBps2NAbWDMoa4x5lCXGG91Y8qqNEnJuiQmWOPGdDe7HFMx5lxf5Sqx81HjEBQbG1vTLzmjdevWKSMjQ506dao6Vl5erl9++UX//e9/VVxcLA+P6rsVe3t7y9v79F9R2Gw2pxmszlQL3ANjDnWNMYe6xHirXfOSj0qShrWP5s/5JMac66rJ9+28Q9CMGTPO67xRo0ad13lXXHGFtmzZUu3YbbfdprZt2+pvf/vbaQEIAAAAjnPseIlW7q3Y3uTKhGiTqwHq1nmHoNGjR5/znJrcExQYGKjExMRqx/z9/RUWFnbacQAAADjWvO1HVG43FB8dpKZh3FsN93LeIchut9dmHQAAAKhDc9ggFW6sxvcE1aZFixaZXQIAAEC9l3+iVEt3ZUqShhKC4IZqvFkqAAAAXNuC5AyVlNvVPNxfLSMCzC4HqHOEIAAAADczN6liKdzQxChZLBaTqwHqHiEIAADAjRSVlGvhydbYdIWDuyIEAQAAuJFfdh1VUWm5GjfwVWJjczebB8xyQSEoJydH77//vh5//HFlZ1f0l1+/fr0OHjzo0OIAAADgWKd2hWMpHNxVjbvDbd68WQMHDlRwcLBSU1N11113KTQ0VNOnT1daWpo++eST2qgTAAAAF6mkzK6ftx+RRGtsuLcazwQ98sgjuvXWW7Vr1y75+PhUHR82bJh++eUXhxYHAAAAx1mxN0v5J8oUHuitzk1DzC4HME2NQ9CaNWt09913n3a8cePGOnz4sEOKAgAAgOPN2ZouSRocHymrlaVwcF81DkHe3t7Ky8s77fjOnTsVHh7ukKIAAADgWOV2Qz8lVSyFG5pIVzi4txqHoFGjRunpp59WaWmpJMlisSgtLU1/+9vfdM011zi8QAAAAFy8NanZyjpeomBfm7o3DzW7HMBUNQ5BL7/8sgoKChQREaGioiL169dPLVu2VGBgoJ599tnaqBEAAAAXqbIr3KD4SNk82CUF7q3G3eGCg4M1b948LV26VJs3b1ZBQYE6deqkgQMH1kZ9AAAAuEh2u6G5SSdbYyfQFQ6ocQiq1KdPH/Xp08eRtQAAAKAWbD6Yq/TcE/L38lCfVg3NLgcwXY1D0H/+858zHrdYLPLx8VHLli3Vt29feXh4XHRxAAAAuHizT3aF6982Qj423qMBNQ5Br776qo4eParCwkKFhFT0lz927Jj8/PwUEBCgjIwMNW/eXAsXLlRMTIzDCwYAAMD5MwxDc0/eD0RXOKBCje+K+9e//qWuXbtq165dysrKUlZWlnbu3Knu3bvr9ddfV1pamqKiovTwww/XRr0AAACogeTD+UrNKpS3p1WXt2E7E0C6gJmgJ554QtOmTVOLFi2qjrVs2VIvvfSSrrnmGu3du1cvvvgi7bIBAACcQGVXuL6tw+XvfcG3gwP1So1ngtLT01VWVnba8bKyMh0+XPGXrFGjRsrPz7/46gAAAHBRKkMQXeGAX9U4BPXv31933323NmzYUHVsw4YNuueeezRgwABJ0pYtWxQXF+e4KgEAAFBje48WaMeRfHlaLRrYLtLscgCnUeMQNHnyZIWGhqpz587y9vaWt7e3unTpotDQUE2ePFmSFBAQoJdfftnhxQIAAOD8zU06Iknq2SJMwX42k6sBnEeNF4ZGRUVp3rx5Sk5O1s6dOyVJbdq0UZs2barO6d+/v+MqBAAAwAWZc7I19pWJLIUDTnXBd8e1bdtWbdu2dWQtAAAAcJCDOUXadCBXFos0OJ4QBJzqgkLQgQMHNGPGDKWlpamkpKTaY6+88opDCgMAAMCFq9wbqGtsqMIDvU2uBnAuNQ5B8+fP16hRo9S8eXMlJycrMTFRqampMgxDnTp1qo0aAQAAUENzkk52hWMpHHCaGjdGePzxx/XYY49py5Yt8vHx0bRp07R//37169dP1113XW3UCAAAgBo4ml+sNanZkqQhhCDgNDUOQdu3b9ctt9wiSfL09FRRUZECAgL09NNP64UXXnB4gQAAAKiZeduOyDCkjk2C1biBr9nlAE6nxiHI39+/6j6g6Oho7dmzp+qxzMxMx1UGAACACzL7ZFc4ZoGAM6vxPUE9evTQ0qVL1a5dOw0bNkyPPvqotmzZounTp6tHjx61USMAAADOU25hqVbsyZIkXZlACALOpMYh6JVXXlFBQYEkadKkSSooKNCXX36pVq1a0RkOAADAZD9vP6Iyu6E2kYFqHh5gdjmAU6pRCCovL9eBAwfUoUMHSRVL4955551aKQwAAAA1R1c44NxqdE+Qh4eHBg8erGPHjtVWPQAAALhAx4vL9MvOo5IIQcDvqXFjhMTERO3du7c2agEAAMBFWLTjqIrL7IoN81PbqECzywGcVo1D0D//+U899thjmjVrltLT05WXl1ftAwAAAOaYuemQpIpZIIvFYnI1gPOqcWOEYcOGSZJGjRpV7S+XYRiyWCwqLy93XHUAAAA4L/uzC/XTtor7ga6+tInJ1QDOrcYhaOHChbVRBwAAAC7CR8tTZTeky1o1VBuWwgG/q8YhqF+/frVRBwAAAC5Q/olSfblmvyTp9j5xJlcDOL8a3xMkSUuWLNFNN92kXr166eDBg5KkTz/9VEuXLnVocQAAADi3r9YeUEFxmVpGBKhfq3CzywGcXo1D0LRp0zRkyBD5+vpq/fr1Ki4uliTl5ubqX//6l8MLBAAAwNmV2w19uCxFknR77zhZrTREAM7lgrrDvfPOO3rvvfdks9mqjvfu3Vvr1693aHEAAAD4fT8lHdaBY0UK8bPp6k6NzS4HcAk1DkE7duxQ3759TzseHBysnJwcR9QEAACA8zR5acUs0I3dY+Vj8zC5GsA11DgERUVFaffu3acdX7p0qZo3b+6QogAAAHBum/bnaO2+Y7J5WHRLz1izywFcRo1D0F133aUHH3xQq1atksVi0aFDhzRlyhQ99thjuueee2qjRgAAAJxB5SzQyI6NFBHkY3I1gOuocYvs//u//5PdbtcVV1yhwsJC9e3bV97e3nrsscd0//3310aNAAAA+I1DOUX6YUu6JOkO2mIDNVLjEGSxWPSPf/xDf/nLX7R7924VFBQoPj5eAQEBtVEfAAAAzuDjFakqtxvq0TxUCY2CzS4HcCk1Xg732WefqbCwUF5eXoqPj1e3bt0IQAAAAHXoeHGZPl+VJkm6ow/3ZAM1VeMQ9PDDDysiIkLjxo3Tjz/+qPLy8tqoCwAAAGcxbf0B5Z0oU7MwP13RNsLscgCXU+MQlJ6eri+++EIWi0Vjx45VdHS07r33Xi1fvrw26gMAAMAp7HZDH5xsiHB7HzZHBS5EjUOQp6enRowYoSlTpigjI0OvvvqqUlNT1b9/f7Vo0aI2agQAAMBJ85MzlJpVqCAfT13TqYnZ5QAuqcaNEU7l5+enIUOG6NixY9q3b5+2b9/uqLoAAABwBpOX7pUk3dC9qfy9L+qtHOC2ajwTJEmFhYWaMmWKhg0bpsaNG+u1117TmDFjlJSU5Oj6AAAAcNLWg7lauTdbnlaLbu3VzOxyAJdV418f/OEPf9CsWbPk5+ensWPH6sknn1TPnj1rozYAAACcovJeoGHtoxUd7GtyNYDrqnEI8vDw0FdffaUhQ4bIw8Oj2mNbt25VYmKiw4oDAABAhYy8E5q5+ZAkNkcFLlaNQ9CUKVOqfZ6fn6/PP/9c77//vtatW0fLbAAAgFrwyYp9Ki031CU2RB1jGphdDuDSLuieIEn65ZdfNH78eEVHR+ull17SgAEDtHLlSkfWBgAAAElFJeWasmqfJOnOy5gFAi5WjWaCDh8+rI8++kiTJ09WXl6exo4dq+LiYn333XeKj4+vrRoBAADc2vQNB3SssFQxob4aFB9ldjmAyzvvmaCRI0eqTZs22rx5s1577TUdOnRIb7zxRm3WBgAA4PZO3Rz11l5x8mBzVOCinfdM0OzZs/XAAw/onnvuUatWrWqzJgAAAJy0eNdR7Tl6XAHenhrbhc1RAUc475mgpUuXKj8/X507d1b37t313//+V5mZmbVZGwAAgNurnAX6Q9cYBfrYTK4GqB/OOwT16NFD7733ntLT03X33Xfriy++UKNGjWS32zVv3jzl5+fXZp0AAABuJ/lwnpbsypTVIo1nc1TAYWrcHc7f31+33367li5dqi1btujRRx/V888/r4iICI0aNao2agQAAHBLlbNAVyZGKSbUz+RqgPrjgltkS1KbNm304osv6sCBA/r8888dVRMAAIDbyywo1ncb2RwVqA0XFYIqeXh4aPTo0ZoxY4Yjng4AAMDtfbZyn0rK7LokpoE6NQ0xuxygXnFICAIAAIDjnCgt16crKjZHvaNPnCwW2mIDjkQIAgAAcDIzNh5S1vESNQr20dBENkcFHI0QBAAA4EQMw9AHyyoaIozv1UyeHrxdAxyNv1UAAABOZNnuLCUfzpefl4f+0K2p2eUA9RIhCAAAwIm8v3SvJGlslxgF+7I5KlAbCEEAAABOYndGvhbtOCqLRbqtdzOzywHqLUIQAACAk/hgWaokaWC7SMWG+ZtbDFCPEYIAAACcQPbxEk1ff0ASm6MCtY0QBAAA4ASmrtqnE6V2JTYOUve4ULPLAeo1QhAAAIDJSsrs+oTNUYE6QwgCAAAw2azNh5SRX6yIQG8Nb9/I7HKAeo8QBAAAYCLDMDR56a+bo3p58vYMqG38LQMAADDRyr3ZSjqUJx+bVePYHBWoE4QgAAAAE1XOAl3TqYlC/L1MrgZwD4QgAAAAk6RmHtf85COSpNtpiw3UGUIQAACAST5cliLDkPq3CVeL8ACzywHcBiEIAADABLmFpfpqbcXmqHde1tzkagD3QggCAAAwwedr0lRUWq62UYHq1SLM7HIAt0IIAgAAqGOl5XZ9vDxVUsW9QGyOCtQtQhAAAEAdm731sNJzT6hhgJdGdWRzVKCuEYIAAADqkGEYmrxkryTp5h7N5GPzMLkiwP2YGoLefvttdejQQUFBQQoKClLPnj01e/ZsM0sCAACoVev2HdOmA7ny8rTqxh5sjgqYwdQQ1KRJEz3//PNat26d1q5dqwEDBuiqq65SUlKSmWUBAADUmsrNUcdc0lgNA7xNrgZwT55mXnzkyJHVPn/22Wf19ttva+XKlUpISDCpKgAAgNqxP7tQc5MOS2JzVMBMpoagU5WXl+vrr7/W8ePH1bNnzzOeU1xcrOLi4qrP8/LyJEmlpaUqLS2tkzrPpvL6ZtcB98GYQ11jzKEu1dfxNnnJHtkNqU/LMDUP86l3r8+V1dcx505q8r2zGIZh1GIt57Rlyxb17NlTJ06cUEBAgKZOnaphw4ad8dyJEydq0qRJpx2fOnWq/Pz8artUAACAC3aiTHpqvYeKyy36U9tytQsx9S0YUO8UFhZq3Lhxys3NVVBQ0O+ea3oIKikpUVpamnJzc/XNN9/o/fff1+LFixUfH3/auWeaCYqJiVFmZuY5X2htKy0t1bx58zRo0CDZbDZTa4F7YMyhrjHmUJfq43j7cPk+/Wv2DrUI99fs+3uxN5CTqY9jzt3k5eWpYcOG5xWCTF8O5+XlpZYtW0qSOnfurDVr1uj111/Xu+++e9q53t7e8vY+/QZCm83mNIPVmWqBe2DMoa4x5lCX6st4Kyu365OVaZKkO/o0l5eXl8kV4Wzqy5hzRzX5vjndPkF2u73abA8AAICr+2nbER04VqQQP5uu7tTY7HIAt2fqTNDjjz+uoUOHqmnTpsrPz9fUqVO1aNEizZ0718yyAAAAHKqyLfZNPWLZHBVwAqaGoIyMDN1yyy1KT09XcHCwOnTooLlz52rQoEFmlgUAAOAwG/fnaN2+Y7J5WHRzj1izywEgk0PQ5MmTzbw8AABAraucBRrZsZEignxMrgaA5IT3BAEAANQXh3KK9OOWdEnSHWyOCjgNQhAAAEAt+Xh5qsrthno2D1NCo2CzywFwEiEIAACgFhwvLtPU1ZVtsZkFApwJIQgAAKAWfLPugPJPlCmuob8GtI0wuxwApyAEAQAAOFi53dCHyyoaItzWu5msVovJFQE4FSEIAADAweZvP6LUrEIF+9p0becmZpcD4DcIQQAAAA5W2Rb7hm5N5edl6o4kAM6AEAQAAOBAWw/malVKtjytFo3vxeaogDMiBAEAADjQBydngYa1j1Z0sK/J1QA4E0IQAACAgxzJO6EZmw5Jku68jLbYgLMiBAEAADjIJytSVWY31LVZiDo0aWB2OQDOghAEAADgAEUl5Zqyis1RAVdACAIAAHCA6RsOKKewVDGhvhoUH2V2OQB+ByEIAADgItntRlVb7Nt6xcmDzVEBp0YIAgAAuEiLdx7V3qPHFejtqbFdY8wuB8A5EIIAAAAuUuUs0PVdYxTgzeaogLMjBAEAAFyE5MN5Wro7U1aLNL5XM7PLAXAeCEEAAAAXYfKSilmgoYnRign1M7kaAOeDEAQAAHCBjuYX6/uNFZuj3k5bbMBlEIIAAAAu0Gcr96mk3K5LYhqoc2yI2eUAOE+EIAAAgAtworRcn63cJ4nNUQFXQwgCAAC4AN9vPKis4yVq3MBXQxPZHBVwJYQgAACAGjKMXzdHHd8rVp4evKUCXAl/YwEAAGpo6e5M7TxSID8vD13ftanZ5QCoIUIQAABADVXOAo3tEqNgX5vJ1QCoKUIQAABADezOyNeiHUdlsUi39W5mdjkALgAhCAAAoAYmL02VJA1qF6nYMH9ziwFwQQhBAAAA5yn7eImmrz8gibbYgCsjBAEAAJynqav2qbjMrsTGQeoWF2p2OQAuECEIAADgPBSXlevjFb9ujmqxWEyuCMCFIgQBAACch1mb0nU0v1iRQd4a3r6R2eUAuAiEIAAAgHM4dXPUW3o2k5cnb6EAV8bfYAAAgHNYuTdb29Lz5GOz6sbubI4KuDpCEAAAwDlMXrpXknRNpyZq4OdlcjUALhYhCAAA4HekZB7X/OQMSdLttMUG6gVCEAAAwO/4cFmKDEMa0DZCLcIDzC4HgAMQggAAAM4it7BUX69lc1SgviEEAQAAnMXU1WkqKi1X26hA9WoRZnY5AByEEAQAAHAGpeV2fbw8VRKbowL1DSEIAADgDH7ckq7DeSfUMMBboy5hc1SgPiEEAQAA/Mapm6Pe3CNW3p4eJlcEwJEIQQAAAL+xdt8xbT6QKy9Pq27sweaoQH1DCAIAAPiNyUsqZoGuvrSxGgZ4m1wNAEcjBAEAAJxif3ahftp2WBKbowL1FSEIAADgFB8uS5XdkC5r1VCtIwPNLgdALSAEAQAAnJR3olRfrkmTxOaoQH1GCAIAADjpqzX7dbykXK0iAtSvdbjZ5QCoJYQgAAAASWXldn24LFVSxb1AbI4K1F+EIAAAAEk/bTuigzlFCvX30phLG5tdDoBaRAgCAACQ9P6SvZKkG7s3lY+NzVGB+owQBAAA3N6GtGNan5YjLw+rbu4Za3Y5AGoZIQgAALi9yUsrNkcd2bGRIgJ9TK4GQG0jBAEAALd2MKdIs7dWbI5KW2zAPRCCAACAW/t4earK7YZ6Ng9TfKMgs8sBUAcIQQAAwG0dLy7T56vZHBVwN4QgAADgtr5eu1/5J8oU19BfA9pGmF0OgDpCCAIAAG6p3G7ow+WpkqTbezeT1crmqIC7IAQBAAC39PP2I9qXVahgX5uu6dzE7HIA1CFCEAAAcEuVbbFv6NZUfl6eJlcDoC4RggAAgNvZejBXq1Oy5Wm1aHwvNkcF3A0hCAAAuJ3KWaDhHaIVHexrcjUA6hohCAAAuJXDuSc0c9MhSbTFBtwVIQgAALiVT1akqsxuqGuzEHVo0sDscgCYgBAEAADcRlFJuaZWbY7a3ORqAJiFEAQAANzGtPUHlFNYqqahfhoUH2l2OQBMQggCAABuwW439MHJhgi39momDzZHBdwWIQgAALiFRTsztDfzuAK9PTW2a4zZ5QAwETuDAQDgAGXldmUXlujY8VJlHS/WseOlyj5erNyiUnVtFqruzcPMLtHtVbbF/kO3GAV48xYIcGf8BABcUNKhXE1duU8H9lvV8ki+EpqEml0SUK8YhqHCknJlHy857SPreImOVf638NfjuUWlv/ucIzpE64nh8YoK9qmjV4FTbU/P07LdWbJapPG9mpldDgCTEYIAF2G3G5qfnKHJS/dq5d7sk0etWvzfFerWLFQ39miqoYnR8vJklSvwW+V2QzmFvwk0hSXKLjg9zFR+FJfZa3wdi0Vq4GtTqL9X1YfVYtHcpMOatTldC5Mz9NDA1rq1dzPZPPi7WpcqZ4GGJkarSYifydUAMBshCHByx4vLNG39AX2wNEWpWYWSJA+rRVcmROrAwUPamuOh1anZWp2arWcCtmlslxiN696Uf+RRrxWVlFeFmOzCEmUfL1b2yeVnZ5q9ySkqlWHU/DpenlaFnRJoQv29FOLnVXEswEuhftUfa+Dndcab7bcezNVT32/V+rQcPfvjdn21dr+evipRPVuwRK4uZOSf0IyNFZuj3s7mqABECAKc1qGcIn28IlWfr0pT3okySVKgj6fGdW+q8T2bKdzfUz/+eECd+lyuaRvS9fnqNB3JK9Zbi/bo7cV7NKBNhG7qGat+rcJlpQMSnJjdbii3qPRkmDm/j6LS8gu6VvApszS/F2YqP/y8PGSxXPzfn8TGwfrmT730zfoDen52snZlFOiG91Zq9CWN9Pdh7RQRxBK52vTZyjSVlNt1adMG6hwbYnY5AJwAIQhwMhv352jy0hT9uCVd5faKX103C/PTbb3jdG3nJvI/eTNvaWnF/QdRQT56aGBr3du/peZvP6LPVqZp6e5MzU/O0PzkDMWE+mpct1iN7dJEYQHepr0uuI/isjPfS3Omj2OFJTpWWFo11mvC5mH5NcwEeCnU31uhfraK//pX/DfE36Ywf++TszQ2U5egWa0Wje0So8HxkXrppx2asipN3208pJ+3Z+jhQa01vmesPFki53AnSss1ZeU+SdIdzAIBOIkQBDiBcruhn5IOa/LSFK3dd6zqeI/mobqjT3MNaBtxzv0sbB5WXZkYrSsTo7X3aIGmrErT12v3a392kV6Yk6xX5+3UsPZRurlnrDo1DXHIb7fhPkrL7dp0IFdbsi06vu6gck+UV1+CVljx32PHS1VQXHZB1wj09qyYlfE/fWYmxN/rtGVpAd6eLjmOG/h56Z+j22tslxg9+X2SNu3P0TOztunrtfv1zOhEdW1GoxNH+m7DQWUdL1HjBr66MiHK7HIAOAlCEGCi/BOl+nLNfn20PFUHjhVJqvjt9siOjXR77zglNg6+oOdtHh6gJ0fE67HBbTRz8yFNWblPmw7k6ruNh/TdxkNqGxWom3vGavQljatmloDfKrcbWpWSpVmb0zV7S7qOFZZK8pB2JJ3zaz2sll+Xm50jzFTO5rhbU48OTRro23t66cu1+/XCnGQlH87Xde+s0NWdGuvxoe0UHsjM7cUyDEMfLKtoiDC+FzNtAH7Fux/ABPuzC/XhslR9tXZ/1W/NQ/xsurF7rG7pGeuw+wN8vTw0tkuMxnaJ0eYDOfps5T59v/GQkg/n6x/fbtVzPybr6k6NdVOPWLWODHTINeHa7HZD69OOadbmdP2wJV1H84urHmvga1OQtURxjcMVFuCtsGqh5tclaKF+Xgrydc1ZmrpmtVp0Q7emujIhSi/O3aEv1qRp+vqDmrftiB4b3EY3dm/KG/eLsGRXpnYeKZC/l4eu79rU7HIAOBFCEFBHDMPQun3HNHlpiuYmHVblLRAtIwJ0e+84jbm0sXy9PGrt+h2aNNCL1zbQP4bF65v1BzRl5T7tzTyuT1bs0ycr9qlbXKhu6hGrKxOi3O438u7OMAxtOZirmZsO6YfN6TqUe6LqsSAfTw1NjNaIjtHqEhOkn+bO0bBhnWSz2UysuP4J8ffSc1e31/VdY/Tkd1u15WCuJsxI0pdrKpbIcTP/halsi31dlxgF+zJmAfyKEATUstJyu37ckq4PlqZo04HcquOXtWqoO/rEqW8dd28L9rPpjj5xur13My3fk6VPV+zTvO1HtDolW6tTstUwwEvXd43RDd1os12fGYah5MP5mrX5kGZuSldadmHVYwHenhoUH6mRHaPVp2V4VSiubMaB2nNJTAN9d29vfb46Tf+eu0Pb0vN0zdvLNbZLE/3tyrY0N6mBXUfytXjnUVks0m29m5ldDgAnQwgCakluYammrk7TJytSlX7yN+tenlaNuaSxbu8TpzZR5i4/s1gs6t2yoXq3bKjDuSf0+eo0fbGmos32mwv36O1FezSgbYRu7EGb7fpkd0aBZm0+pFmb07U7o6DquI/NqivaRWpkh0a6vE24fGy1NyuJ3+dhteimHrEamhilF+Yk66u1B/TV2gOas/Ww/nJlW43r1vScjVKgqnuBBrWLVGyYv8nVAHA2hCDAwVIyj+vDZSn6eu2Bqr1MGgZ46eYezXRjj6Zq6IS/yY0K9tHDg1rrvgEt9fO2I/ps1T4t252ln7dn6OftGWoa6qdx3ZtqbJcYhfp7mV0uaigtq1AzTwaf7el5Vce9PKy6vE24RnRspCvaRtAkw8mEBXjrxWs76vquTfXkd1u1LT1PT363VV+dXCJ3SUwDs0t0WtnHSzR9/UFJtMUGcGb8iwc4gGEYWrE3Sx8sTdH85IyqnenbRgXq9j5xGtWxkUv8Zt3mYdXQ9tEa2j5ae44WaMrKNH2zbr/Ssgv1/OxkvTJvp4a3j9ZNPWLVqWkDbnx3Yum5Rfphc7pmbjpUbRmmp9WiPq0aamSHRhqUEKkgH+6TcHadY0M0477emrIqTS/9tENbDuZqzFvL9IeuMfrrkLYK4RcTp5mycp+Ky+xq3zhY3eJoOQ7gdIQg4CKUlNk1c9MhTV6aom2n/IZ9QNsI3dEnTr1ahLlsUGgRHqCnRsbrL0PaaOamQ/ps1T5tPpCrbzcc1LcbDqpddJBu7hGrqy5pxAyCk8jIP6HZWw5r1uZDWpP6635TVovUs0WYRnRopCsTonjT7II8Pawa36uZhrWP1vOzkzVt/QF9vnq/Zm89rL9d2VbXd4lhyepJxWXl+njFr5ujuurPYAC1i3cuwAXIPl6iKSv36ZOV+6paCPvYrLqmUxPd1jtOLSMCTK7QcXy9PDS2a4zGdo3Rpv0VbbZnbDqk7el5+vu3W/SvH7frmpNttlvRZrvOHTteotlbK4LPyr1ZVV0HJalbs1CN6BitoYnR7DlTT4QHeuvlsR11fdcYPfX9ViUfztfj07foizX79c+rEtW+yYXtLVafzNyUrsyCYkUGeWtY+2izywHgpAhBQA3szsjX5KWpmr7+gIrL7JKkyCBv3dKzmcZ1a1rvf8PeMaaBOsY00D+Gt9M36w5oyqo0pWQe18cr9unjk222b+4RqyG02a5VeSdK9VPSEc3cdEjLdmeq7JTk0zGmgUZ2iNbwDtGKDvY1sUrUpm5xoZp1fx99vGKfXp23U5v252jUm0t1Y/ememxwGzXwq98/i87GMIyqtti39GzGzyEAZ2VqCHruuec0ffp0JScny9fXV7169dILL7ygNm3amFkWUI1hGFqyK1OTl6Zo8c6jVccTGwfpzj7NNax9tNv9Q9vAz0t3XtZct/eO0/I9Wfps5W/bbHvrD11jdEP3pmrcgDfijnC8uEw/bz+iWZvTtXjHUZWU26sei48O0oiO0RrRvpGahtHW3F14elh1R584jewQrX/9uF3fbTykz1am6ccth/V/V7bVtZ2buN0SuRV7s7Q9PU++Ng/d2J3NUQGcnakhaPHixbr33nvVtWtXlZWV6e9//7sGDx6sbdu2yd+fdpYw14nScn234aA+WJainUcqWglbLBXtVu/oE6ducaFuv9bcevIm+z6tfm2z/fnqNGXkF+u/C3frrUW7NaBtpG7q0bTO90OqD06UlmvRjgzN3JSu+clHdKL01+DTMiJAIzs00oiO0WoRXn+WX6LmIoJ89NofLtX1XZvqqe+3aldGgf46bbO+WJOmZ0YnKqGR+yyRm7ykYhboms6N3XY2DMD5MTUEzZkzp9rnH330kSIiIrRu3Tr17dv3tPOLi4tVXFxc9XleXsWN6KWlpaZv4ld5fbPrwMXLLCjWlFX7NXXNfmUfr/h++nl56NpOjXVLz6aKDa34TXtZWZmZZTrdmAvz89B9l8fp7stiNT/5qKau3q8Ve7P18/Yj+nn7EcWE+OqGbk10zaWNabP9O0rK7Fq6J0s/bjmsn7dn6HhJedVjTUN9Nbx9lIYnRql1ZEBVCK+rMeBsYw7VdWkapO//3EOfrEzTGwv2aH1ajka+UbFE7qEBLRTk61qdAGs63lIyj2t+coYk6eZuMYxT1Bg/41xfTb53FsMwjHOfVjd2796tVq1aacuWLUpMTDzt8YkTJ2rSpEmnHZ86dar8/FgCgotz8Li0KN2qdZkWlRsVby5DvAz1jbarR4QhP+6gq7EjRdKyw1atPmpRUXnFn6mnxdClYYZ6R9nVLKBids3dlRvSrlyL1mdatDn71z8rqWIMXhpmqFNDu5r48+eF85NTLH23z6oNWRVLdQNshq6KtatrQ6PejqGv91q19IhV8Q3surud/dxfAKDeKSws1Lhx45Sbm6ugoKDfPddpQpDdbteoUaOUk5OjpUuXnvGcM80ExcTEKDMz85wvtLaVlpZq3rx5GjRokGw21/ptmzuz2w0t3pWpj5bv0/K92VXHL4kJ1m09YzU4PkKeHs55v48rjbnCkjL9sOWwpq4+oK2Hfm0l3i4qUOO6xWhkhyi3a7Ndbje0dt8x/bDlsOZuO1I16yhJ4QFeGpoYpeHto3RJk2CnWUboSmMOFZbvydKkWcnam3lcktQltoEmjminNlHO38mxJuMtp7BUfV9arKJSuz65rbN6Ng+roypRn/AzzvXl5eWpYcOG5xWCnOZdx7333qutW7eeNQBJkre3t7y9T2/zarPZnGawOlMtOLvCkjJNW39QHy5L0d6jFW8OrBZpaGK0bu8Tp86xISZXeP5cYcwF22wa1yNO43rEadP+HH26cp9mbjqk7Yfz9eSMbXpx7k5d7QZttg3D0Pq0HM3afEg/bE5XRv6vv9QJ9a8IPiM6NFK3uFB5OEnwORNXGHOo0K9tlOa0jNDkpSn6z/xdWrsvR1e9vVLjezbTw4NaKdAFNss9n/H29YZ9Kiq1q21UoC5rHen292vi4vAzznXV5PvmFCHovvvu06xZs/TLL7+oSZMmZpeDeuxw7gl9siJVU1alKbeo4jfvgd6e+kO3GI3v1UxNQlhWWdsq22w/cZY2293jQnVzz1gNjq8fbbYNw9DWg3matfmQZm1O18GcoqrHgnw8NSQhSiM7NlKvFmFOO+sI1+bladU9l7fQVZc00j9/2KYftxzWB8tSNHPzIT0xvJ1GdWzk0qGhtNyuT5azOSqAmjE1BBmGofvvv1/ffvutFi1apLi4ODPLQT225UCuJi/dq1mb06v2VIkJ9dXtveN0XZcYBbjZUixncGqb7WV7MivabG87olUp2Vp1ss32Dd1idEO3pmrkgm22dxzO18xNhzRr8yGlZhVWHff38tCg+EiN6NBIl7VuKG9PDxOrhDtp1MBXb93YWb/sPKoJM5KUknlcD36xUVNXVXSRa+2is7A/bknX4bwTahjgrVGXNDK7HAAuwtR3fvfee6+mTp2q77//XoGBgTp8+LAkKTg4WL6+rvemB86l3G5o3rYj+mBpilan/nq/T7dmobq9T5wGxUc69ZIjd2G1WnRZq3Bd1ipc6blF+nz1fn1xss32Gwt2682FFW22b+4Zq8taNnSa+2POZO/RAs3anK6Zmw5pV0ZB1XEfm1VXtI3UiA7R6t82Qj42gg/M07d1uOY8dJneX5KiNxbs0qqUbA17fYlu691MDw5s7VK/FKq+OWosv1QAcN5M/Un39ttvS5Iuv/zyasc//PBD3XrrrXVfEOqFguIyfbVmvz5anqq07IrfwHtaLRrRIVp39Gmu9k3cZ88MVxMd7KtHBrXW/QNaat62I/ps5T4t35NV1WY7NsxPN3Zvqus6xyjESdps788u1KzN6Zq1+ZCSTmn64OVhVd/W4RrZMVoD20W6XeMHODdvTw/d27+lRnVspGdmbdNP247ovSUpmrHpkJ4YHq8RHaJdYlnZmtRj2nwgV16eVjZHBVAjpi+HAxzlwLFCfbw8VV+s3q/84oo9fIJ9bRrXvanG92ymqGAfkyvE+bJ5WDWsfbSGtY/W7owCTVm1T9+sO6B9WYX614/JeumnnRrRIVo39YjVpTEN6vzN2uHcE1X3+Gzcn1N13MNqUZ+WDTWiQ7QGJ0Qp2MX2ZYH7iQn10/9u6aKFyRmaODNJ+7IKdf/nG/TFmjRNGpWolhHOvRHv5KV7JUlXX9pYYQGnN04CgLPhV5Nweev2HdMHS1M0J+mwyk/e79O8ob9u6xOnazo1lp8Xw9yVtYwI0ISRCfrLkDaauemQPl25T1sP5mn6+oOavv6gEhoF6aYesbrqkka1+r3OLCjW7C3pmrkpXWv2ZavydzgWi9SzeZhGdGikKxOj2AgWLql/2wj1bBGmdxfv1VuLdmvZ7iwNff0X3dGnuR64oqVT/hxNyyrUT9uOSJJu78M9xQBqxvl+qgHnoazcrjlJhzV5aYo2pOVUHe/VIkx39IlT/zYRTn3vCGrOz8tT13dtqrFdYrTpQK4+XbGvagna49O36F8/bNc1nZvoph5N1TLCMTd45xSWaM7Ww5q1OV3L92TKfsrkdZfYEI3s2EhD20cpIpBZRrg+H5uHHhzYSmMubaxJM5M0PzlD7yzeoxkbD+rJEfG6MjHKqZbIfbg8RYZRcY+TqzZ1AGAeQhBcSm5Rqb5ck6aPl++rajXs5WHVqEsa6fbecYpvZO6muah9FotFl8Q00CUxDfTkiIo225+t3KfUrEJ9tDxVHy1PVY/mobqpx4W12c4/Uap5245o5qZDWrIrs6qboCR1bBKsER0aaXiHaJfsWAecj6Zhfpp8a1f9vO2IJs5M0oFjRbpnynpd1qqhJo1KUPNw85fI5Z0o1Vdr9kuqaIsNADVFCIJL2Jd1XB8uS9XXa/freEm5pIrNJW/qEaubejTlN/Fu6rdttj9dsU8/bz+ilXuztXJvtsIDvfWHrudus11YUqb52zM0c9MhLdp5VCVl9qrH2kYFamTHRhrRIVqxYf518bIApzAwPlJ9WjXUWwt3653Fe7VkV6aufG2J/ti3ue7t31K+XuZ1YvtydcW/Ba0iAtS3VUPT6gDgughBcFqGYWh1SrYmL03RvO1Hqu7BaBURoDv6xGn0pY1pNQxJ1dtsH8op0her0/T5mv06ekqb7SvaRermHrHqc7LN9onSci3acVSzNh/S/O0ZKiotr3q+FuH+J4NPI6e/MRyoTT42Dz0yuI2u7tREE2YkafHOo/rvwt36dsNBPTUyXoPjI+t8iVxZuV0fLU+VVHEvkDMt0QPgOghBcDolZXb9sOWQJi9N0daDv7Yc7tc6XHf0idNlrRryjx7OqlEDXz0yuI3uv6KVfkqqaLO9Ym+W5m07onnbKtpst28crEU7jqrgZBdBSWoa6qcRHaI1smMjtY0KZIwBp2jW0F8f3dZVP207oqdnbtPBnCLd/ek69W8TromjEup0lnRu0hEdzClSqL+XxlzauM6uC6B+IQTBaeQUlmjKqjR9siJVR/KKJUnenlZd3amxbu8dp1bc+IoasHlYNbxDtIZ3iNbujHx9tjJN09ZXtNnel1Wxf1SjYB8N7xCtER0aqUOTYIIP8DssFouGJETpslYN9ebC3frfL3u1cMdRLXv1F/2pXwv9+fIWdTI7//7Jttg3dW/KagAAF4wQBFOVlNmVknlcn6xI1bT1B3SitOJejPBAb93SI1Y39oil5TAuWsuIQE0claC/XlnRZnt/dpEubxOuTk1D6CII1JCfl6f+MqStru7URBNnJGnJrkz9Z/4ufbvhgCaOTNAV7SJr7drr045pQ1qOvDysuqlnbK1dB0D9RwiCwxWXlSuzoESZ+cXKLKj8KNHR/GIdLSg+5XiJcotKq31tfHSQ7ugTpxEdo+XtyW/44FiVbbYBXLwW4QH65PZumr31sJ6ZtU37s4t0x8drNbBdhCaMTFBMqJ/Drzl5aYokaWTHRjTEAXBRCEE4LydKy6uCS+YZwszRyrCTX6y8E2XnfsJT2DwsJ+/3aa4ezUNZkgQALsJisWhY+2j1ax2u/yzYpclLUvTz9gwt2ZWpe/u31B/7NnfYkrWDOUWas/WwJNpiA7h4hCA3dqK0/DezMyWnzNwU62j+r6Env7jmwaZhgPfJD6+K/wZWfB4eWHEs/OTjwb42liQBgAvz9/bU40Pb6brOTfTkd0lasTdLr8zbqenrD2jiqARd3ibioq/x8fJUldsN9WoRxp5wAC4aIaieKSwpU2b+KTMzBcUnPz+hzPySasvTCmoYbLw8rBWBpjLMBHirYaDXKWHHW+EnPw/2tTGjAwBupmVEoKbe1V0zN6frn7O2KTWrULd+uEZDEiL15Ih4NQm5sCVyBcVl+nxVmiRmgQA4BiHIBRwvLqs2O3P0DPfbVD5WWFJ+7ic8hZen9WSY8VZ4gFf12ZtqMzfeCvLxJNgAAH6XxWLRqI6NNKBthF7/eac+WJaquUlHtHjnUd0/oJXuvCyuxvd8fr12v/KLy9S8ob/6O2BWCQAIQSYwDEPHS8pPLjf79d6ao6eEmVNncU7dxPF8+NisZ5ydqQwzp4acQG+CDQDA8QK8PfWP4fG6tnOMnvx+q1anZOvfc3do2roDmnRVgi5rFX5ez1NuN/ThslRJ0m29m7F8GoBDEIIcxDAMFZVJKZnHlXPCflqYOZpf/X6bylbQ58vX5lG19Cz81PtrznC/jb+XB8EGAOAU2kQF6ss/9tD3Gw/pnz9s197M47p58moNbx+tJ0a0U3Sw7+9+/YLko0rLLlSwr03XdG5SR1UDqO8IQQ7ypykbtWCHp7Rm2Xl/jZ+XxymzM6csRQusCDrhp9xv4+/NtwoA4JosFotGX9pYA9pF6NV5O/Xx8lT9sCVdC3dk6IErWun23nHy8rSe8Ws/WJ4qSRrXvan8vPi3EIBj8NPEQYL9bJIkf2+Pqq5nDU82DggP8KnWQKCyoQA/zAEA7iTIx6YJIxN0XecYPfX9Vq3dd0zPz07WN+sO6OlRCerVsmG18/cXSGv35cjTatH4ns3MKRpAvcS7cAd5Ymgb9bKlafTIwbLZbGaXAwCA04pvFKSv7u6p6RsO6rkft2t3RoHGvb9KIzs20j+GtVNUcMVGqAvTK2aHhneIrjoGAI5w5rln1FiQr01ejtkPDgCAes9qtejazk204LHLNb5nrKwWaeamQ7ri5UV6f8leHThWpA1ZFfe30hYbgKMRggAAgGmCfW2adFWiZtzXR5c2baDjJeX65w/bNfy/y2U3LOoS20AdmjQwu0wA9QwhCAAAmC6xcbCm/amXXrymg0L9var2vbutV6zJlQGoj7gnCAAAOAWr1aKxXWM0OCFS/12wS3v37NUVbdkcFYDjMRMEAACcSgM/L/1tSGtd1cwuDzZHBVALCEEAAAAA3AohCAAAAIBbIQQBAAAAcCuEIAAAAABuhRAEAAAAwK0QggAAAAC4FUIQAAAAALdCCAIAAADgVghBAAAAANwKIQgAAACAWyEEAQAAAHArhCAAAAAAboUQBAAAAMCtEIIAAAAAuBVCEAAAAAC3QggCAAAA4FYIQQAAAADcCiEIAAAAgFvxNLuAi2EYhiQpLy/P5Eqk0tJSFRYWKi8vTzabzexy4AYYc6hrjDnUJcYb6hpjzvVVZoLKjPB7XDoE5efnS5JiYmJMrgQAAACAM8jPz1dwcPDvnmMxzicqOSm73a5Dhw4pMDBQFovF1Fry8vIUExOj/fv3KygoyNRa4B4Yc6hrjDnUJcYb6hpjzvUZhqH8/Hw1atRIVuvv3/Xj0jNBVqtVTZo0MbuMaoKCgviLgzrFmENdY8yhLjHeUNcYc67tXDNAlWiMAAAAAMCtEIIAAAAAuBVCkIN4e3trwoQJ8vb2NrsUuAnGHOoaYw51ifGGusaYcy8u3RgBAAAAAGqKmSAAAAAAboUQBAAAAMCtEIIAAAAAuBVCEAAAAAC3Qgg6xXPPPaeuXbsqMDBQERERGj16tHbs2FHtnBMnTujee+9VWFiYAgICdM011+jIkSPVznnggQfUuXNneXt765JLLvnda+7evVuBgYFq0KCBg18NnF1djjfDMPTSSy+pdevW8vb2VuPGjfXss8/W1kuDk6rLMTd37lz16NFDgYGBCg8P1zXXXKPU1NRaemVwVo4Yc5s2bdINN9ygmJgY+fr6ql27dnr99ddPu9aiRYvUqVMneXt7q2XLlvroo49q++XBydTVeJs+fboGDRqk8PBwBQUFqWfPnpo7d26dvEY4DiHoFIsXL9a9996rlStXat68eSotLdXgwYN1/PjxqnMefvhhzZw5U19//bUWL16sQ4cO6eqrrz7tuW6//XZdf/31v3u90tJS3XDDDbrssssc/lrg/OpyvD344IN6//339dJLLyk5OVkzZsxQt27dauV1wXnV1ZhLSUnRVVddpQEDBmjjxo2aO3euMjMzz/g8qN8cMebWrVuniIgIffbZZ0pKStI//vEPPf744/rvf/9bdU5KSoqGDx+u/v37a+PGjXrooYd055138sbUzdTVePvll180aNAg/fjjj1q3bp369++vkSNHasOGDXX6enGRDJxVRkaGIclYvHixYRiGkZOTY9hsNuPrr7+uOmf79u2GJGPFihWnff2ECROMjh07nvX5//rXvxo33XST8eGHHxrBwcGOLh8uprbG27Zt2wxPT08jOTm51mqHa6qtMff1118bnp6eRnl5edWxGTNmGBaLxSgpKXH8C4HLuNgxV+nPf/6z0b9//6rP//rXvxoJCQnVzrn++uuNIUOGOPgVwJXU1ng7k/j4eGPSpEmOKRx1gpmg35GbmytJCg0NlVTx24HS0lINHDiw6py2bduqadOmWrFiRY2ee8GCBfr666/15ptvOq5guLTaGm8zZ85U8+bNNWvWLMXFxalZs2a68847lZ2d7dgXAJdTW2Ouc+fOslqt+vDDD1VeXq7c3Fx9+umnGjhwoGw2m2NfBFyKo8Zcbm5u1XNI0ooVK6o9hyQNGTKkxv82o36prfH2W3a7Xfn5+b97DpwPIegs7Ha7HnroIfXu3VuJiYmSpMOHD8vLy+u0+3ciIyN1+PDh837urKws3Xrrrfroo48UFBTkyLLhompzvO3du1f79u3T119/rU8++UQfffSR1q1bp2uvvdaRLwEupjbHXFxcnH766Sf9/e9/l7e3txo0aKADBw7oq6++cuRLgItx1Jhbvny5vvzyS/3xj3+sOnb48GFFRkae9hx5eXkqKipy7AuBS6jN8fZbL730kgoKCjR27FiH1Y/a52l2Ac7q3nvv1datW7V06VKHP/ddd92lcePGqW/fvg5/brim2hxvdrtdxcXF+uSTT9S6dWtJ0uTJk9W5c2ft2LFDbdq0cfg14fxqc8wdPnxYd911l8aPH68bbrhB+fn5euqpp3Tttddq3rx5slgsDr8mnJ8jxtzWrVt11VVXacKECRo8eLADq0N9U1fjberUqZo0aZK+//57RUREXPC1UPeYCTqD++67T7NmzdLChQvVpEmTquNRUVEqKSlRTk5OtfOPHDmiqKio837+BQsW6KWXXpKnp6c8PT11xx13KDc3V56envrggw8c9TLgImp7vEVHR8vT07MqAElSu3btJElpaWkXVzxcUm2PuTfffFPBwcF68cUXdemll6pv37767LPPNH/+fK1atcpRLwMuxBFjbtu2bbriiiv0xz/+UU888US1x6Kiok7rYnjkyBEFBQXJ19fXsS8GTq+2x1ulL774Qnfeeae++uqr05ZjwvkRgk5hGIbuu+8+ffvtt1qwYIHi4uKqPd65c2fZbDbNnz+/6tiOHTuUlpamnj17nvd1VqxYoY0bN1Z9PP300woMDNTGjRs1ZswYh70eOLe6Gm+9e/dWWVmZ9uzZU3Vs586dkqTY2NiLfBVwJXU15goLC2W1Vv/nxcPDQ1LFzCTch6PGXFJSkvr376/x48efsb1/z549qz2HJM2bN69G4xaur67GmyR9/vnnuu222/T5559r+PDhtfOCULtMbcvgZO655x4jODjYWLRokZGenl71UVhYWHXOn/70J6Np06bGggULjLVr1xo9e/Y0evbsWe15du3aZWzYsMG4++67jdatWxsbNmwwNmzYYBQXF5/xunSHc091Nd7Ky8uNTp06GX379jXWr19vrF271ujevbsxaNCgOn29MF9djbn58+cbFovFmDRpkrFz505j3bp1xpAhQ4zY2Nhq10L954gxt2XLFiM8PNy46aabqj1HRkZG1Tl79+41/Pz8jL/85S/G9u3bjTfffNPw8PAw5syZU6evF+aqq/E2ZcoUw9PT03jzzTernZOTk1OnrxcXhxB0Ckln/Pjwww+rzikqKjL+/Oc/GyEhIYafn58xZswYIz09vdrz9OvX74zPk5KScsbrEoLcU12Ot4MHDxpXX321ERAQYERGRhq33nqrkZWVVUevFM6iLsfc559/blx66aWGv7+/ER4ebowaNcrYvn17Hb1SOAtHjLkJEyac8TliY2OrXWvhwoXGJZdcYnh5eRnNmzevdg24h7oab2f7GTh+/Pi6e7G4aBbDMAzHzCkBAAAAgPPjniAAAAAAboUQBAAAAMCtEIIAAAAAuBVCEAAAAAC3QggCAAAA4FYIQQAAAADcCiEIAAAAgFshBAEAAABwK4QgAAAAAG6FEAQAcBqGYWjgwIEaMmTIaY+99dZbatCggQ4cOGBCZQCA+oQQBABwGhaLRR9++KFWrVqld999t+p4SkqK/vrXv+qNN95QkyZNHHrN0tJShz4fAMD5EYIAAE4lJiZGr7/+uh577DGlpKTIMAzdcccdGjx4sC699FINHTpUAQEBioyM1M0336zMzMyqr50zZ4769OmjBg0aKCwsTCNGjNCePXuqHk9NTZXFYtGXX36pfv36ycfHR1OmTDHjZQIATGQxDMMwuwgAAH5r9OjRys3N1dVXX61nnnlGSUlJSkhI0J133qlbbrlFRUVF+tvf/qaysjItWLBAkjRt2jRZLBZ16NBBBQUFeuqpp5SamqqNGzfKarUqNTVVcXFxatasmV5++WVdeuml8vHxUXR0tMmvFgBQlwhBAACnlJGRoYSEBGVnZ2vatGnaunWrlixZorlz51adc+DAAcXExGjHjh1q3br1ac+RmZmp8PBwbdmyRYmJiVUh6LXXXtODDz5Yly8HAOBEWA4HAHBKERERuvvuu9WuXTuNHj1amzZt0sKFCxUQEFD10bZtW0mqWvK2a9cu3XDDDWrevLmCgoLUrFkzSVJaWlq15+7SpUudvhYAgHPxNLsAAADOxtPTU56eFf9UFRQUaOTIkXrhhRdOO69yOdvIkSMVGxur9957T40aNZLdbldiYqJKSkqqne/v71/7xQMAnBYhCADgEjp16qRp06apWbNmVcHoVFlZWdqxY4fee+89XXbZZZKkpUuX1nWZAAAXwHI4AIBLuPfee5Wdna0bbrhBa9as0Z49ezR37lzddtttKi8vV0hIiMLCwvS///1Pu3fv1oIFC/TII4+YXTYAwAkRggAALqFRo0ZatmyZysvLNXjwYLVv314PPfSQGjRoIKvVKqvVqi+++ELr1q1TYmKiHn74Yf373/82u2wAgBOiOxwAAAAAt8JMEAAAAAC3QggCAAAA4FYIQQAAAADcCiEIAAAAgFshBAEAAABwK4QgAAAAAG6FEAQAAADArRCCAAAAALgVQhAAAAAAt0IIAgAAAOBWCEEAAAAA3Mr/A0VAtdI/K4eiAAAAAElFTkSuQmCC\n" - }, - "metadata": {} - } - ], - "source": [ - "import pandas as pd\n", - "import matplotlib.pyplot as plt\n", - "\n", - "# Load data\n", - "df = pd.read_csv(\"/tmp/tmpvzjigv7g/n2OzlTWhinflation.csv\")\n", - "\n", - "# Calculate average yearly inflation\n", - "df['Average'] = df[['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']].mean(axis=1)\n", - "\n", - "# Plot average yearly inflation as a time series\n", - "plt.figure(figsize=(10,6))\n", - "plt.plot(df['Year'], df['Average'])\n", - "plt.title('Average Yearly Inflation')\n", - "plt.xlabel('Year')\n", - "plt.ylabel('Average Inflation')\n", - "plt.grid(True)\n", - "plt.show()" - ], - "id": "JqBBVLKdIHHq" - }, - { - "cell_type": "markdown", - "id": "FJ85DUhgBZd7", - "metadata": { - "id": "FJ85DUhgBZd7" - }, - "source": [ - "## 3. Llama Stack Agent Evaluations\n" - ] - }, - { - "cell_type": "markdown", - "id": "ydeBDpDT5VHd", - "metadata": { - "id": "ydeBDpDT5VHd" - }, - "source": [ - "#### 3.1. Online Evaluation Dataset Collection Using Telemetry\n", - "\n", - "- Llama Stack offers built-in telemetry to collect traces and data about your agentic application.\n", - "- In this example, we will show how to build an Agent with Llama Stack, and query the agent's traces into an online dataset that can be used for evaluation. " - ] - }, - { - "cell_type": "markdown", - "id": "_t_tcWq0JcJ4", - "metadata": { - "id": "_t_tcWq0JcJ4" - }, - "source": [ - "##### 3.1.1. Building a Search Agent" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "id": "4iCO59kP20Zs", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "4iCO59kP20Zs", - "outputId": "894c6333-30e9-4f1e-9b63-1bfb1cae51e2" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "inference> brave_search.call(query=\"NBA Western Conference Finals 2024 teams\")\n", - "tool_execution> Tool:brave_search Args:{'query': 'NBA Western Conference Finals 2024 teams'}\n", - "tool_execution> Tool:brave_search Response:{\"query\": \"NBA Western Conference Finals 2024 teams\", \"top_k\": [{\"title\": \"2024 NBA Western Conference Finals - Basketball-Reference.com\", \"url\": \"https://www.basketball-reference.com/playoffs/2024-nba-western-conference-finals-mavericks-vs-timberwolves.html\", \"content\": \"2024 NBA Western Conference Finals Mavericks vs. Timberwolves League Champion: Boston Celtics. Finals MVP: Jaylen Brown (20.8 / 5.4 / 5.0) 2024 Playoff Leaders: PTS: Luka Don\\u010di\\u0107 (635) TRB: Luka Don\\u010di\\u0107 (208) AST: Luka Don\\u010di\\u0107 (178) WS: Derrick White (2.9) More playoffs info\", \"score\": 0.9310187, \"raw_content\": null}, {\"title\": \"NBA Western Conference Finals 2024: Dates, schedule and more - Sportskeeda\", \"url\": \"https://www.sportskeeda.com/basketball/news-nba-western-conference-finals-2024-dates-schedule-and-more\", \"content\": \"NBA Western Conference Finals 2024: Dates & Schedule The 2023-24 NBA Western Conference Finals will start on Wednesday, May 22. The Mavericks will face the team that wins in Game 7 between the\", \"score\": 0.8914433, \"raw_content\": null}, {\"title\": \"2024 Playoffs: West Finals | Timberwolves (3) vs. Mavericks (5) - NBA.com\", \"url\": \"https://www.nba.com/playoffs/2024/west-final\", \"content\": \"The Dallas Mavericks and Minnesota Timberwolves have advanced to the 2024 Western Conference Finals during the NBA playoffs.\", \"score\": 0.8884594, \"raw_content\": null}, {\"title\": \"NBA Conference Finals Schedule: Full List of Games & Results\", \"url\": \"https://www.si.com/nba/nba-conference-finals-schedule-full-list-of-games-results\", \"content\": \"The 2024 NBA conference finals matchups are set. Here's the schedule for all the games. ... Western Conference First Round (1) Oklahoma City Thunder def. (8) New Orleans Pelicans in 4 games\", \"score\": 0.85008353, \"raw_content\": null}, {\"title\": \"NBA Finals 2024 - Celtics-Mavericks news, schedule, scores and ... - ESPN\", \"url\": \"https://www.espn.com/nba/story/_/id/39943302/nba-playoffs-2024-conference-finals-news-scores-highlights\", \"content\": \"The Boston Celtics are the 2024 NBA Champions. ... Western Conference. Final 2023-24 NBA regular-season standings. Which team left standing has the most trips to the NBA Finals? Here is a look at\", \"score\": 0.81979275, \"raw_content\": null}]}\n", - "inference> The teams that played in the NBA Western Conference Finals of 2024 were the Dallas Mavericks and the Minnesota Timberwolves.\n", - "inference> brave_search.call(query=\"Bill Cosby South Park episode\")\n", - "tool_execution> Tool:brave_search Args:{'query': 'Bill Cosby South Park episode'}\n", - "tool_execution> Tool:brave_search Response:{\"query\": \"Bill Cosby South Park episode\", \"top_k\": [{\"title\": \"Bill Cosby | South Park Archives | Fandom\", \"url\": \"https://southpark.fandom.com/wiki/Bill_Cosby\", \"content\": \"For other uses, see Bill (Disambiguation). William Henry \\\"Bill\\\" Cosby Jr. African-American comedian, actor, and serial rapist. He first appears in the Season Five episode, \\\"Here Comes the Neighborhood\\\", as one of the wealthy African-Americans who move to South Park. He returned as a hologram in the Season Eighteen episode, \\\"#HappyHolograms\\\" where he is shown trying to molest pop star Taylor\", \"score\": 0.82288796, \"raw_content\": null}, {\"title\": \"Trapper Keeper (South Park) - Wikipedia\", \"url\": \"https://en.wikipedia.org/wiki/Trapper_Keeper_(South_Park)\", \"content\": \"Bill Cosby warns that if the Trapper Keeper assimilates with the supercomputer at Cheyenne Mountain, it will become unstoppable. ... It is one of the many South Park episodes that parodies a current event. [1] The main plot of the episode involving the Trapper Keeper was written before the election, [1]\", \"score\": 0.75659186, \"raw_content\": null}, {\"title\": \"Bill Cosby is Here to See You - South Park Studios US\", \"url\": \"https://southpark.cc.com/video-clips/wfot8s/south-park-bill-cosby-is-here-to-see-you\", \"content\": \"Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. ... South Park. Bill Cosby is Here to See You. Season 18 E 10 \\u2022 12/10/2014. Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. More. Watch Random Episode. Watching. 01:11. Please Welcome \\\"Cartman Bra\\\" South Park S18 E9.\", \"score\": 0.7156829, \"raw_content\": null}, {\"title\": \"Bill Cosby and Taylor Swift Duet - South Park Studios\", \"url\": \"https://www.southparkstudios.com/video-clips/90r7i1/south-park-bill-cosby-and-taylor-swift-duet\", \"content\": \"The holiday special continues with Bill Cosby and Taylor Swift's rendition of \\\"It's Snowing Out There\\\". ... Full Episodes. Collections. Random Episode. Full Episodes. Events. Wiki. News. Avatar. Shop. Forum. Games. South Park. Menu. Episodes & Videos. About. South Park. Bill Cosby and Taylor Swift Duet. Season 18 E 10 \\u2022 12/10/2014. The\", \"score\": 0.64639384, \"raw_content\": null}, {\"title\": \"Bill Cosby (android) | South Park Character ... - South Park Studios US\", \"url\": \"https://southpark.cc.com/wiki/Bill_Cosby_(android)\", \"content\": \"About. Sent back in time to destroy Eric Cartman's Dawson's Creek Trapper Keeper before it manifests into an omnipotent supercomputer that can destroy all humanity, \\\"Bill Cosby\\\" is really VSM471, an android or cyborg of some kind engineered by 'hoomans' in the distant future. He fails in his initial missions to infiltrate South Park Elementary's 4th Grade class, destroy the Trapper Keeper or\", \"score\": 0.56460327, \"raw_content\": null}]}\n", - "inference> Bill Cosby (BSM-471) first appears in the Season 4 episode \"Trapper Keeper\" of South Park.\n", - "inference> brave_search.call(query=\"Andrew Tate kickboxing name\")\n", - "tool_execution> Tool:brave_search Args:{'query': 'Andrew Tate kickboxing name'}\n", - "tool_execution> Tool:brave_search Response:{\"query\": \"Andrew Tate kickboxing name\", \"top_k\": [{\"title\": \"50 Facts About Andrew Tate - Facts.net\", \"url\": \"https://facts.net/andrew-tate-facts/\", \"content\": \"Full Name: Andrew Tate's full name is Emory Andrew Tate III, named after his father, a celebrated chess player. Date of Birth: ... Kickboxing Start: Tate began his kickboxing career in 2005, starting his journey as a professional fighter, which would later be a significant part of his persona. First Championship:\", \"score\": 0.8967681, \"raw_content\": null}, {\"title\": \"The Life Of Andrew Tate (By Andrew Tate Himself)\", \"url\": \"https://sidekickboxing.co.uk/the-life-of-andrew-king-cobra-tate/\", \"content\": \"Andrew Tate stats. Fight Name: Cobra Tate. Born: 1 December 1986. Weight: 90 KG. Weight Class: Cruiserweight. Height: 1.92m. Fight Record: Wins - 76, Losses - 9. ... Andrew Tate's Kickboxing Career. Andrew Tate has always fought credible opponents right from the beginning of his kickboxing career. One of his first professional fights on\", \"score\": 0.8795718, \"raw_content\": null}, {\"title\": \"About Andrew Tate | The Real World\", \"url\": \"https://www.taterealworldofficial.com/about-andrew-tate\", \"content\": \"Emory Andrew Tate III (born December 14, 1986) is an American-British kickboxer from Chicago, Illinois, who competes in the cruiserweight and heavyweight divisions. ... Tate challenged Paul Randall for the vacant ISKA English Kickboxing Light-cruiserweight title. Tate won his first ISKA Kickboxing title stopping Randall in the fifth round of\", \"score\": 0.8386933, \"raw_content\": null}, {\"title\": \"Andrew Tate - Fight Record - Muay Thai Records\", \"url\": \"https://muaythairecords.com/fighters/andrew-tate\", \"content\": \"Andrew \\\"King Cobra\\\" Tate is a 38-year-old Muay Thai fighter. With a record of 23-8-0, including 32 knockouts, standing at 6\\u2032 4\\u2033 and weighing 198 lbs. Originally from Luton, United Kingdom. ... WIN Dec -Kickboxing Jean Luc Beno\\u00eet. 14th Mar 2015 -Boxe in D\\u00e9fi 16. Andrew Tate defeated Jean Luc Beno\\u00eet by decision. ... Name: Andrew Tate\", \"score\": 0.8194462, \"raw_content\": null}, {\"title\": \"Andrew Tate: Kickboxing Record, Facts, Height, Weight, Age, Biography\", \"url\": \"https://www.lowkickmma.com/andrew-tate-kickboxing-record-facts-height-weight-age-biography/\", \"content\": \"Birth Name: Emory Andrew Tate III: Date of Birth: 1 December 1986: Place of Birth: Washington, D.C., U.S. ... In his professional kickboxing career, Andrew Tate won 32 of his fights by knockout.\", \"score\": 0.7992077, \"raw_content\": null}]}\n", - "inference> Andrew Tate's kickboxing name is \"King Cobra\" or \"Cobra Tate\".\n" - ] - } - ], - "source": [ - "from llama_stack_client.lib.agents.agent import Agent\n", - "from llama_stack_client.lib.agents.event_logger import EventLogger\n", - "from llama_stack_client.types.agent_create_params import AgentConfig\n", - "from google.colab import userdata\n", - "\n", - "agent_config = AgentConfig(\n", - " model=\"meta-llama/Llama-3.1-405B-Instruct-FP8\",\n", - " instructions=\"You are a helpful assistant. Use search tool to answer the questions. \",\n", - " toolgroups=[\"builtin::websearch\"],\n", - " input_shields=[],\n", - " output_shields=[],\n", - " enable_session_persistence=False,\n", - ")\n", - "agent = Agent(client, agent_config)\n", - "user_prompts = [\n", - " \"Which teams played in the NBA western conference finals of 2024\",\n", - " \"In which episode and season of South Park does Bill Cosby (BSM-471) first appear? Give me the number and title.\",\n", - " \"What is the British-American kickboxer Andrew Tate's kickboxing name?\",\n", - "]\n", - "\n", - "session_id = agent.create_session(\"test-session\")\n", - "\n", - "for prompt in user_prompts:\n", - " response = agent.create_turn(\n", - " messages=[\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": prompt,\n", - " }\n", - " ],\n", - " session_id=session_id,\n", - " )\n", - "\n", - " for log in EventLogger().log(response):\n", - " log.print()" - ] - }, - { - "cell_type": "markdown", - "id": "ekOS2kM4P0LM", - "metadata": { - "id": "ekOS2kM4P0LM" - }, - "source": [ - "##### 3.1.2 Query Telemetry" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "id": "agkWgToGAsuA", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 1000 - }, - "id": "agkWgToGAsuA", - "outputId": "4233a1d9-8282-4aa9-bdc4-0c105939f97e" - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Getting traces for session_id=44d006af-1394-4832-9799-5f0cb0ca01d6\n" - ] - }, - { - "output_type": "display_data", - "data": { - "text/plain": [ - "\u001b[1m[\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[1m{\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'input'\u001b[0m: \u001b[1m[\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"system\",\"content\":\"You are a helpful assistant. Use search tool to answer the questions. \"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"call_id\":\"b7d9e0dd-4d6d-47db-9d81-3d7834f6e53d\",\"tool_name\":\"brave_search\",\"arguments\":\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"query\":\"NBA Western Conference Finals 2024 teams\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"ipython\",\"call_id\":\"b7d9e0dd-4d6d-47db-9d81-3d7834f6e53d\",\"tool_name\":\"brave_search\",\"content\":\"\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"query\\\\\": \\\\\"NBA Western Conference Finals 2024 teams\\\\\", \\\\\"top_k\\\\\": \u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference Finals - Basketball-Reference.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.basketball-reference.com/playoffs/2024-nba-western-conference-finals-mavericks-vs-timberwolves.html\\\\\", \\\\\"content\\\\\": \\\\\"2024 NBA Western Conference Finals Mavericks vs. Timberwolves League Champion: Boston Celtics. Finals MVP: Jaylen Brown \u001b[0m\u001b[32m(\u001b[0m\u001b[32m20.8 / 5.4 / 5.0\u001b[0m\u001b[32m)\u001b[0m\u001b[32m 2024 Playoff Leaders: PTS: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m635\u001b[0m\u001b[32m)\u001b[0m\u001b[32m TRB: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m208\u001b[0m\u001b[32m)\u001b[0m\u001b[32m AST: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m178\u001b[0m\u001b[32m)\u001b[0m\u001b[32m WS: Derrick White \u001b[0m\u001b[32m(\u001b[0m\u001b[32m2.9\u001b[0m\u001b[32m)\u001b[0m\u001b[32m More playoffs info\\\\\", \\\\\"score\\\\\": 0.9310187, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates, schedule and more - Sportskeeda\\\\\", \\\\\"url\\\\\": \\\\\"https://www.sportskeeda.com/basketball/news-nba-western-conference-finals-2024-dates-schedule-and-more\\\\\", \\\\\"content\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates & Schedule The 2023-24 NBA Western Conference Finals will start on Wednesday, May 22. The Mavericks will face the team that wins in Game 7 between the\\\\\", \\\\\"score\\\\\": 0.8914433, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"2024 Playoffs: West Finals | Timberwolves \u001b[0m\u001b[32m(\u001b[0m\u001b[32m3\u001b[0m\u001b[32m)\u001b[0m\u001b[32m vs. Mavericks \u001b[0m\u001b[32m(\u001b[0m\u001b[32m5\u001b[0m\u001b[32m)\u001b[0m\u001b[32m - NBA.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.nba.com/playoffs/2024/west-final\\\\\", \\\\\"content\\\\\": \\\\\"The Dallas Mavericks and Minnesota Timberwolves have advanced to the 2024 Western Conference Finals during the NBA playoffs.\\\\\", \\\\\"score\\\\\": 0.8884594, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"NBA Conference Finals Schedule: Full List of Games & Results\\\\\", \\\\\"url\\\\\": \\\\\"https://www.si.com/nba/nba-conference-finals-schedule-full-list-of-games-results\\\\\", \\\\\"content\\\\\": \\\\\"The 2024 NBA conference finals matchups are set. Here\\'s the schedule for all the games. ... Western Conference First Round \u001b[0m\u001b[32m(\u001b[0m\u001b[32m1\u001b[0m\u001b[32m)\u001b[0m\u001b[32m Oklahoma City Thunder def. \u001b[0m\u001b[32m(\u001b[0m\u001b[32m8\u001b[0m\u001b[32m)\u001b[0m\u001b[32m New Orleans Pelicans in 4 games\\\\\", \\\\\"score\\\\\": 0.85008353, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"NBA Finals 2024 - Celtics-Mavericks news, schedule, scores and ... - ESPN\\\\\", \\\\\"url\\\\\": \\\\\"https://www.espn.com/nba/story/_/id/39943302/nba-playoffs-2024-conference-finals-news-scores-highlights\\\\\", \\\\\"content\\\\\": \\\\\"The Boston Celtics are the 2024 NBA Champions. ... Western Conference. Final 2023-24 NBA regular-season standings. Which team left standing has the most trips to the NBA Finals? Here is a look at\\\\\", \\\\\"score\\\\\": 0.81979275, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"The teams that played in the NBA Western Conference Finals of 2024 were the Dallas Mavericks and the Minnesota Timberwolves.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"user\",\"content\":\"In which episode and season of South Park does Bill Cosby \u001b[0m\u001b[32m(\u001b[0m\u001b[32mBSM-471\u001b[0m\u001b[32m)\u001b[0m\u001b[32m first appear? Give me the number and title.\",\"context\":null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"call_id\":\"1e487e8e-a15f-4137-854a-1d4979a70b8c\",\"tool_name\":\"brave_search\",\"arguments\":\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"query\":\"Bill Cosby South Park episode\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"ipython\",\"call_id\":\"1e487e8e-a15f-4137-854a-1d4979a70b8c\",\"tool_name\":\"brave_search\",\"content\":\"\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"query\\\\\": \\\\\"Bill Cosby South Park episode\\\\\", \\\\\"top_k\\\\\": \u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Bill Cosby | South Park Archives | Fandom\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.fandom.com/wiki/Bill_Cosby\\\\\", \\\\\"content\\\\\": \\\\\"For other uses, see Bill \u001b[0m\u001b[32m(\u001b[0m\u001b[32mDisambiguation\u001b[0m\u001b[32m)\u001b[0m\u001b[32m. William Henry \\\\\\\\\\\\\"Bill\\\\\\\\\\\\\" Cosby Jr. African-American comedian, actor, and serial rapist. He first appears in the Season Five episode, \\\\\\\\\\\\\"Here Comes the Neighborhood\\\\\\\\\\\\\", as one of the wealthy African-Americans who move to South Park. He returned as a hologram in the Season Eighteen episode, \\\\\\\\\\\\\"#HappyHolograms\\\\\\\\\\\\\" where he is shown trying to molest pop star Taylor\\\\\", \\\\\"score\\\\\": 0.82288796, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Trapper Keeper \u001b[0m\u001b[32m(\u001b[0m\u001b[32mSouth Park\u001b[0m\u001b[32m)\u001b[0m\u001b[32m - Wikipedia\\\\\", \\\\\"url\\\\\": \\\\\"https://en.wikipedia.org/wiki/Trapper_Keeper_\u001b[0m\u001b[32m(\u001b[0m\u001b[32mSouth_Park\u001b[0m\u001b[32m)\u001b[0m\u001b[32m\\\\\", \\\\\"content\\\\\": \\\\\"Bill Cosby warns that if the Trapper Keeper assimilates with the supercomputer at Cheyenne Mountain, it will become unstoppable. ... It is one of the many South Park episodes that parodies a current event. \u001b[0m\u001b[32m[\u001b[0m\u001b[32m1\u001b[0m\u001b[32m]\u001b[0m\u001b[32m The main plot of the episode involving the Trapper Keeper was written before the election, \u001b[0m\u001b[32m[\u001b[0m\u001b[32m1\u001b[0m\u001b[32m]\u001b[0m\u001b[32m\\\\\", \\\\\"score\\\\\": 0.75659186, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Bill Cosby is Here to See You - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/video-clips/wfot8s/south-park-bill-cosby-is-here-to-see-you\\\\\", \\\\\"content\\\\\": \\\\\"Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. ... South Park. Bill Cosby is Here to See You. Season 18 E 10 \\\\\\\\u2022 12/10/2014. Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. More. Watch Random Episode. Watching. 01:11. Please Welcome \\\\\\\\\\\\\"Cartman Bra\\\\\\\\\\\\\" South Park S18 E9.\\\\\", \\\\\"score\\\\\": 0.7156829, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Bill Cosby and Taylor Swift Duet - South Park Studios\\\\\", \\\\\"url\\\\\": \\\\\"https://www.southparkstudios.com/video-clips/90r7i1/south-park-bill-cosby-and-taylor-swift-duet\\\\\", \\\\\"content\\\\\": \\\\\"The holiday special continues with Bill Cosby and Taylor Swift\\'s rendition of \\\\\\\\\\\\\"It\\'s Snowing Out There\\\\\\\\\\\\\". ... Full Episodes. Collections. Random Episode. Full Episodes. Events. Wiki. News. Avatar. Shop. Forum. Games. South Park. Menu. Episodes & Videos. About. South Park. Bill Cosby and Taylor Swift Duet. Season 18 E 10 \\\\\\\\u2022 12/10/2014. The\\\\\", \\\\\"score\\\\\": 0.64639384, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Bill Cosby \u001b[0m\u001b[32m(\u001b[0m\u001b[32mandroid\u001b[0m\u001b[32m)\u001b[0m\u001b[32m | South Park Character ... - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/wiki/Bill_Cosby_\u001b[0m\u001b[32m(\u001b[0m\u001b[32mandroid\u001b[0m\u001b[32m)\u001b[0m\u001b[32m\\\\\", \\\\\"content\\\\\": \\\\\"About. Sent back in time to destroy Eric Cartman\\'s Dawson\\'s Creek Trapper Keeper before it manifests into an omnipotent supercomputer that can destroy all humanity, \\\\\\\\\\\\\"Bill Cosby\\\\\\\\\\\\\" is really VSM471, an android or cyborg of some kind engineered by \\'hoomans\\' in the distant future. He fails in his initial missions to infiltrate South Park Elementary\\'s 4th Grade class, destroy the Trapper Keeper or\\\\\", \\\\\"score\\\\\": 0.56460327, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"Bill Cosby \u001b[0m\u001b[32m(\u001b[0m\u001b[32mBSM-471\u001b[0m\u001b[32m)\u001b[0m\u001b[32m first appears in the Season 4 episode \\\\\"Trapper Keeper\\\\\" of South Park.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"user\",\"content\":\"What is the British-American kickboxer Andrew Tate\\'s kickboxing name?\",\"context\":null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[1m]\u001b[0m,\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'output'\u001b[0m: \u001b[32m\"content: tool_calls: \u001b[0m\u001b[32m[\u001b[0m\u001b[32mToolCall\u001b[0m\u001b[32m(\u001b[0m\u001b[32mcall_id\u001b[0m\u001b[32m='44705eaf-b371-4841-b0ee-5eb21a5d7f36', \u001b[0m\u001b[32mtool_name\u001b[0m\u001b[32m=\u001b[0m\u001b[32m<\u001b[0m\u001b[32mBuiltinTool.brave_search:\u001b[0m\u001b[32m 'brave_search'>, \u001b[0m\u001b[32marguments\u001b[0m\u001b[32m=\u001b[0m\u001b[32m{\u001b[0m\u001b[32m'query': 'Andrew Tate kickboxing name'\u001b[0m\u001b[32m}\u001b[0m\u001b[32m)\u001b[0m\u001b[32m]\u001b[0m\u001b[32m\"\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[1;39m{\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'input'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"call_id\":\"44705eaf-b371-4841-b0ee-5eb21a5d7f36\",\"tool_name\":\"brave_search\",\"arguments\":\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"query\":\"Andrew Tate kickboxing name\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'output'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"ipython\",\"call_id\":\"44705eaf-b371-4841-b0ee-5eb21a5d7f36\",\"tool_name\":\"brave_search\",\"content\":\"\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"query\\\\\": \\\\\"Andrew Tate kickboxing name\\\\\", \\\\\"top_k\\\\\": \u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"50 Facts About Andrew Tate - Facts.net\\\\\", \\\\\"url\\\\\": \\\\\"https://facts.net/andrew-tate-facts/\\\\\", \\\\\"content\\\\\": \\\\\"Full Name: Andrew Tate\\'s full name is Emory Andrew Tate III, named after his father, a celebrated chess player. Date of Birth: ... Kickboxing Start: Tate began his kickboxing career in 2005, starting his journey as a professional fighter, which would later be a significant part of his persona. First Championship:\\\\\", \\\\\"score\\\\\": 0.8967681, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"The Life Of Andrew Tate \u001b[0m\u001b[32m(\u001b[0m\u001b[32mBy Andrew Tate Himself\u001b[0m\u001b[32m)\u001b[0m\u001b[32m\\\\\", \\\\\"url\\\\\": \\\\\"https://sidekickboxing.co.uk/the-life-of-andrew-king-cobra-tate/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate stats. Fight Name: Cobra Tate. Born: 1 December 1986. Weight: 90 KG. Weight Class: Cruiserweight. Height: 1.92m. Fight Record: Wins - 76, Losses - 9. ... Andrew Tate\\'s Kickboxing Career. Andrew Tate has always fought credible opponents right from the beginning of his kickboxing career. One of his first professional fights on\\\\\", \\\\\"score\\\\\": 0.8795718, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"About Andrew Tate | The Real World\\\\\", \\\\\"url\\\\\": \\\\\"https://www.taterealworldofficial.com/about-andrew-tate\\\\\", \\\\\"content\\\\\": \\\\\"Emory Andrew Tate III \u001b[0m\u001b[32m(\u001b[0m\u001b[32mborn December 14, 1986\u001b[0m\u001b[32m)\u001b[0m\u001b[32m is an American-British kickboxer from Chicago, Illinois, who competes in the cruiserweight and heavyweight divisions. ... Tate challenged Paul Randall for the vacant ISKA English Kickboxing Light-cruiserweight title. Tate won his first ISKA Kickboxing title stopping Randall in the fifth round of\\\\\", \\\\\"score\\\\\": 0.8386933, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Andrew Tate - Fight Record - Muay Thai Records\\\\\", \\\\\"url\\\\\": \\\\\"https://muaythairecords.com/fighters/andrew-tate\\\\\", \\\\\"content\\\\\": \\\\\"Andrew \\\\\\\\\\\\\"King Cobra\\\\\\\\\\\\\" Tate is a 38-year-old Muay Thai fighter. With a record of 23-8-0, including 32 knockouts, standing at 6\\\\\\\\u2032 4\\\\\\\\u2033 and weighing 198 lbs. Originally from Luton, United Kingdom. ... WIN Dec -Kickboxing Jean Luc Beno\\\\\\\\u00eet. 14th Mar 2015 -Boxe in D\\\\\\\\u00e9fi 16. Andrew Tate defeated Jean Luc Beno\\\\\\\\u00eet by decision. ... Name: Andrew Tate\\\\\", \\\\\"score\\\\\": 0.8194462, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Andrew Tate: Kickboxing Record, Facts, Height, Weight, Age, Biography\\\\\", \\\\\"url\\\\\": \\\\\"https://www.lowkickmma.com/andrew-tate-kickboxing-record-facts-height-weight-age-biography/\\\\\", \\\\\"content\\\\\": \\\\\"Birth Name: Emory Andrew Tate III: Date of Birth: 1 December 1986: Place of Birth: Washington, D.C., U.S. ... In his professional kickboxing career, Andrew Tate won 32 of his fights by knockout.\\\\\", \\\\\"score\\\\\": 0.7992077, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[1;39m{\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'input'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;39m[\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"system\",\"content\":\"You are a helpful assistant. Use search tool to answer the questions. \"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"call_id\":\"b7d9e0dd-4d6d-47db-9d81-3d7834f6e53d\",\"tool_name\":\"brave_search\",\"arguments\":\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"query\":\"NBA Western Conference Finals 2024 teams\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"ipython\",\"call_id\":\"b7d9e0dd-4d6d-47db-9d81-3d7834f6e53d\",\"tool_name\":\"brave_search\",\"content\":\"\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"query\\\\\": \\\\\"NBA Western Conference Finals 2024 teams\\\\\", \\\\\"top_k\\\\\": \u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference Finals - Basketball-Reference.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.basketball-reference.com/playoffs/2024-nba-western-conference-finals-mavericks-vs-timberwolves.html\\\\\", \\\\\"content\\\\\": \\\\\"2024 NBA Western Conference Finals Mavericks vs. Timberwolves League Champion: Boston Celtics. Finals MVP: Jaylen Brown \u001b[0m\u001b[32m(\u001b[0m\u001b[32m20.8 / 5.4 / 5.0\u001b[0m\u001b[32m)\u001b[0m\u001b[32m 2024 Playoff Leaders: PTS: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m635\u001b[0m\u001b[32m)\u001b[0m\u001b[32m TRB: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m208\u001b[0m\u001b[32m)\u001b[0m\u001b[32m AST: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m178\u001b[0m\u001b[32m)\u001b[0m\u001b[32m WS: Derrick White \u001b[0m\u001b[32m(\u001b[0m\u001b[32m2.9\u001b[0m\u001b[32m)\u001b[0m\u001b[32m More playoffs info\\\\\", \\\\\"score\\\\\": 0.9310187, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates, schedule and more - Sportskeeda\\\\\", \\\\\"url\\\\\": \\\\\"https://www.sportskeeda.com/basketball/news-nba-western-conference-finals-2024-dates-schedule-and-more\\\\\", \\\\\"content\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates & Schedule The 2023-24 NBA Western Conference Finals will start on Wednesday, May 22. The Mavericks will face the team that wins in Game 7 between the\\\\\", \\\\\"score\\\\\": 0.8914433, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"2024 Playoffs: West Finals | Timberwolves \u001b[0m\u001b[32m(\u001b[0m\u001b[32m3\u001b[0m\u001b[32m)\u001b[0m\u001b[32m vs. Mavericks \u001b[0m\u001b[32m(\u001b[0m\u001b[32m5\u001b[0m\u001b[32m)\u001b[0m\u001b[32m - NBA.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.nba.com/playoffs/2024/west-final\\\\\", \\\\\"content\\\\\": \\\\\"The Dallas Mavericks and Minnesota Timberwolves have advanced to the 2024 Western Conference Finals during the NBA playoffs.\\\\\", \\\\\"score\\\\\": 0.8884594, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"NBA Conference Finals Schedule: Full List of Games & Results\\\\\", \\\\\"url\\\\\": \\\\\"https://www.si.com/nba/nba-conference-finals-schedule-full-list-of-games-results\\\\\", \\\\\"content\\\\\": \\\\\"The 2024 NBA conference finals matchups are set. Here\\'s the schedule for all the games. ... Western Conference First Round \u001b[0m\u001b[32m(\u001b[0m\u001b[32m1\u001b[0m\u001b[32m)\u001b[0m\u001b[32m Oklahoma City Thunder def. \u001b[0m\u001b[32m(\u001b[0m\u001b[32m8\u001b[0m\u001b[32m)\u001b[0m\u001b[32m New Orleans Pelicans in 4 games\\\\\", \\\\\"score\\\\\": 0.85008353, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"NBA Finals 2024 - Celtics-Mavericks news, schedule, scores and ... - ESPN\\\\\", \\\\\"url\\\\\": \\\\\"https://www.espn.com/nba/story/_/id/39943302/nba-playoffs-2024-conference-finals-news-scores-highlights\\\\\", \\\\\"content\\\\\": \\\\\"The Boston Celtics are the 2024 NBA Champions. ... Western Conference. Final 2023-24 NBA regular-season standings. Which team left standing has the most trips to the NBA Finals? Here is a look at\\\\\", \\\\\"score\\\\\": 0.81979275, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"The teams that played in the NBA Western Conference Finals of 2024 were the Dallas Mavericks and the Minnesota Timberwolves.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"user\",\"content\":\"In which episode and season of South Park does Bill Cosby \u001b[0m\u001b[32m(\u001b[0m\u001b[32mBSM-471\u001b[0m\u001b[32m)\u001b[0m\u001b[32m first appear? Give me the number and title.\",\"context\":null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"call_id\":\"1e487e8e-a15f-4137-854a-1d4979a70b8c\",\"tool_name\":\"brave_search\",\"arguments\":\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"query\":\"Bill Cosby South Park episode\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"ipython\",\"call_id\":\"1e487e8e-a15f-4137-854a-1d4979a70b8c\",\"tool_name\":\"brave_search\",\"content\":\"\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"query\\\\\": \\\\\"Bill Cosby South Park episode\\\\\", \\\\\"top_k\\\\\": \u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Bill Cosby | South Park Archives | Fandom\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.fandom.com/wiki/Bill_Cosby\\\\\", \\\\\"content\\\\\": \\\\\"For other uses, see Bill \u001b[0m\u001b[32m(\u001b[0m\u001b[32mDisambiguation\u001b[0m\u001b[32m)\u001b[0m\u001b[32m. William Henry \\\\\\\\\\\\\"Bill\\\\\\\\\\\\\" Cosby Jr. African-American comedian, actor, and serial rapist. He first appears in the Season Five episode, \\\\\\\\\\\\\"Here Comes the Neighborhood\\\\\\\\\\\\\", as one of the wealthy African-Americans who move to South Park. He returned as a hologram in the Season Eighteen episode, \\\\\\\\\\\\\"#HappyHolograms\\\\\\\\\\\\\" where he is shown trying to molest pop star Taylor\\\\\", \\\\\"score\\\\\": 0.82288796, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Trapper Keeper \u001b[0m\u001b[32m(\u001b[0m\u001b[32mSouth Park\u001b[0m\u001b[32m)\u001b[0m\u001b[32m - Wikipedia\\\\\", \\\\\"url\\\\\": \\\\\"https://en.wikipedia.org/wiki/Trapper_Keeper_\u001b[0m\u001b[32m(\u001b[0m\u001b[32mSouth_Park\u001b[0m\u001b[32m)\u001b[0m\u001b[32m\\\\\", \\\\\"content\\\\\": \\\\\"Bill Cosby warns that if the Trapper Keeper assimilates with the supercomputer at Cheyenne Mountain, it will become unstoppable. ... It is one of the many South Park episodes that parodies a current event. \u001b[0m\u001b[32m[\u001b[0m\u001b[32m1\u001b[0m\u001b[32m]\u001b[0m\u001b[32m The main plot of the episode involving the Trapper Keeper was written before the election, \u001b[0m\u001b[32m[\u001b[0m\u001b[32m1\u001b[0m\u001b[32m]\u001b[0m\u001b[32m\\\\\", \\\\\"score\\\\\": 0.75659186, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Bill Cosby is Here to See You - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/video-clips/wfot8s/south-park-bill-cosby-is-here-to-see-you\\\\\", \\\\\"content\\\\\": \\\\\"Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. ... South Park. Bill Cosby is Here to See You. Season 18 E 10 \\\\\\\\u2022 12/10/2014. Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. More. Watch Random Episode. Watching. 01:11. Please Welcome \\\\\\\\\\\\\"Cartman Bra\\\\\\\\\\\\\" South Park S18 E9.\\\\\", \\\\\"score\\\\\": 0.7156829, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Bill Cosby and Taylor Swift Duet - South Park Studios\\\\\", \\\\\"url\\\\\": \\\\\"https://www.southparkstudios.com/video-clips/90r7i1/south-park-bill-cosby-and-taylor-swift-duet\\\\\", \\\\\"content\\\\\": \\\\\"The holiday special continues with Bill Cosby and Taylor Swift\\'s rendition of \\\\\\\\\\\\\"It\\'s Snowing Out There\\\\\\\\\\\\\". ... Full Episodes. Collections. Random Episode. Full Episodes. Events. Wiki. News. Avatar. Shop. Forum. Games. South Park. Menu. Episodes & Videos. About. South Park. Bill Cosby and Taylor Swift Duet. Season 18 E 10 \\\\\\\\u2022 12/10/2014. The\\\\\", \\\\\"score\\\\\": 0.64639384, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Bill Cosby \u001b[0m\u001b[32m(\u001b[0m\u001b[32mandroid\u001b[0m\u001b[32m)\u001b[0m\u001b[32m | South Park Character ... - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/wiki/Bill_Cosby_\u001b[0m\u001b[32m(\u001b[0m\u001b[32mandroid\u001b[0m\u001b[32m)\u001b[0m\u001b[32m\\\\\", \\\\\"content\\\\\": \\\\\"About. Sent back in time to destroy Eric Cartman\\'s Dawson\\'s Creek Trapper Keeper before it manifests into an omnipotent supercomputer that can destroy all humanity, \\\\\\\\\\\\\"Bill Cosby\\\\\\\\\\\\\" is really VSM471, an android or cyborg of some kind engineered by \\'hoomans\\' in the distant future. He fails in his initial missions to infiltrate South Park Elementary\\'s 4th Grade class, destroy the Trapper Keeper or\\\\\", \\\\\"score\\\\\": 0.56460327, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"Bill Cosby \u001b[0m\u001b[32m(\u001b[0m\u001b[32mBSM-471\u001b[0m\u001b[32m)\u001b[0m\u001b[32m first appears in the Season 4 episode \\\\\"Trapper Keeper\\\\\" of South Park.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"user\",\"content\":\"What is the British-American kickboxer Andrew Tate\\'s kickboxing name?\",\"context\":null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"call_id\":\"44705eaf-b371-4841-b0ee-5eb21a5d7f36\",\"tool_name\":\"brave_search\",\"arguments\":\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"query\":\"Andrew Tate kickboxing name\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"ipython\",\"call_id\":\"44705eaf-b371-4841-b0ee-5eb21a5d7f36\",\"tool_name\":\"brave_search\",\"content\":\"\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"query\\\\\": \\\\\"Andrew Tate kickboxing name\\\\\", \\\\\"top_k\\\\\": \u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"50 Facts About Andrew Tate - Facts.net\\\\\", \\\\\"url\\\\\": \\\\\"https://facts.net/andrew-tate-facts/\\\\\", \\\\\"content\\\\\": \\\\\"Full Name: Andrew Tate\\'s full name is Emory Andrew Tate III, named after his father, a celebrated chess player. Date of Birth: ... Kickboxing Start: Tate began his kickboxing career in 2005, starting his journey as a professional fighter, which would later be a significant part of his persona. First Championship:\\\\\", \\\\\"score\\\\\": 0.8967681, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"The Life Of Andrew Tate \u001b[0m\u001b[32m(\u001b[0m\u001b[32mBy Andrew Tate Himself\u001b[0m\u001b[32m)\u001b[0m\u001b[32m\\\\\", \\\\\"url\\\\\": \\\\\"https://sidekickboxing.co.uk/the-life-of-andrew-king-cobra-tate/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate stats. Fight Name: Cobra Tate. Born: 1 December 1986. Weight: 90 KG. Weight Class: Cruiserweight. Height: 1.92m. Fight Record: Wins - 76, Losses - 9. ... Andrew Tate\\'s Kickboxing Career. Andrew Tate has always fought credible opponents right from the beginning of his kickboxing career. One of his first professional fights on\\\\\", \\\\\"score\\\\\": 0.8795718, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"About Andrew Tate | The Real World\\\\\", \\\\\"url\\\\\": \\\\\"https://www.taterealworldofficial.com/about-andrew-tate\\\\\", \\\\\"content\\\\\": \\\\\"Emory Andrew Tate III \u001b[0m\u001b[32m(\u001b[0m\u001b[32mborn December 14, 1986\u001b[0m\u001b[32m)\u001b[0m\u001b[32m is an American-British kickboxer from Chicago, Illinois, who competes in the cruiserweight and heavyweight divisions. ... Tate challenged Paul Randall for the vacant ISKA English Kickboxing Light-cruiserweight title. Tate won his first ISKA Kickboxing title stopping Randall in the fifth round of\\\\\", \\\\\"score\\\\\": 0.8386933, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Andrew Tate - Fight Record - Muay Thai Records\\\\\", \\\\\"url\\\\\": \\\\\"https://muaythairecords.com/fighters/andrew-tate\\\\\", \\\\\"content\\\\\": \\\\\"Andrew \\\\\\\\\\\\\"King Cobra\\\\\\\\\\\\\" Tate is a 38-year-old Muay Thai fighter. With a record of 23-8-0, including 32 knockouts, standing at 6\\\\\\\\u2032 4\\\\\\\\u2033 and weighing 198 lbs. Originally from Luton, United Kingdom. ... WIN Dec -Kickboxing Jean Luc Beno\\\\\\\\u00eet. 14th Mar 2015 -Boxe in D\\\\\\\\u00e9fi 16. Andrew Tate defeated Jean Luc Beno\\\\\\\\u00eet by decision. ... Name: Andrew Tate\\\\\", \\\\\"score\\\\\": 0.8194462, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Andrew Tate: Kickboxing Record, Facts, Height, Weight, Age, Biography\\\\\", \\\\\"url\\\\\": \\\\\"https://www.lowkickmma.com/andrew-tate-kickboxing-record-facts-height-weight-age-biography/\\\\\", \\\\\"content\\\\\": \\\\\"Birth Name: Emory Andrew Tate III: Date of Birth: 1 December 1986: Place of Birth: Washington, D.C., U.S. ... In his professional kickboxing career, Andrew Tate won 32 of his fights by knockout.\\\\\", \\\\\"score\\\\\": 0.7992077, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[1;39m]\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'output'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'content: Andrew Tate\\'s kickboxing name is \"King Cobra\" or \"Cobra Tate\". tool_calls: \u001b[0m\u001b[32m[\u001b[0m\u001b[32m]\u001b[0m\u001b[32m'\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[1;39m{\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'input'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;39m[\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"system\",\"content\":\"You are a helpful assistant. Use search tool to answer the questions. \"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[1;39m]\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'output'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m\"content: tool_calls: \u001b[0m\u001b[32m[\u001b[0m\u001b[32mToolCall\u001b[0m\u001b[32m(\u001b[0m\u001b[32mcall_id\u001b[0m\u001b[32m='b7d9e0dd-4d6d-47db-9d81-3d7834f6e53d', \u001b[0m\u001b[32mtool_name\u001b[0m\u001b[32m=, \u001b[0m\u001b[32marguments\u001b[0m\u001b[32m=\u001b[0m\u001b[32m{\u001b[0m\u001b[32m'query': 'NBA Western Conference Finals 2024 teams'\u001b[0m\u001b[32m}\u001b[0m\u001b[32m)\u001b[0m\u001b[32m]\u001b[0m\u001b[32m\"\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[1;39m{\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'input'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"call_id\":\"b7d9e0dd-4d6d-47db-9d81-3d7834f6e53d\",\"tool_name\":\"brave_search\",\"arguments\":\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"query\":\"NBA Western Conference Finals 2024 teams\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'output'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"ipython\",\"call_id\":\"b7d9e0dd-4d6d-47db-9d81-3d7834f6e53d\",\"tool_name\":\"brave_search\",\"content\":\"\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"query\\\\\": \\\\\"NBA Western Conference Finals 2024 teams\\\\\", \\\\\"top_k\\\\\": \u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference Finals - Basketball-Reference.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.basketball-reference.com/playoffs/2024-nba-western-conference-finals-mavericks-vs-timberwolves.html\\\\\", \\\\\"content\\\\\": \\\\\"2024 NBA Western Conference Finals Mavericks vs. Timberwolves League Champion: Boston Celtics. Finals MVP: Jaylen Brown \u001b[0m\u001b[32m(\u001b[0m\u001b[32m20.8 / 5.4 / 5.0\u001b[0m\u001b[32m)\u001b[0m\u001b[32m 2024 Playoff Leaders: PTS: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m635\u001b[0m\u001b[32m)\u001b[0m\u001b[32m TRB: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m208\u001b[0m\u001b[32m)\u001b[0m\u001b[32m AST: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m178\u001b[0m\u001b[32m)\u001b[0m\u001b[32m WS: Derrick White \u001b[0m\u001b[32m(\u001b[0m\u001b[32m2.9\u001b[0m\u001b[32m)\u001b[0m\u001b[32m More playoffs info\\\\\", \\\\\"score\\\\\": 0.9310187, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates, schedule and more - Sportskeeda\\\\\", \\\\\"url\\\\\": \\\\\"https://www.sportskeeda.com/basketball/news-nba-western-conference-finals-2024-dates-schedule-and-more\\\\\", \\\\\"content\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates & Schedule The 2023-24 NBA Western Conference Finals will start on Wednesday, May 22. The Mavericks will face the team that wins in Game 7 between the\\\\\", \\\\\"score\\\\\": 0.8914433, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"2024 Playoffs: West Finals | Timberwolves \u001b[0m\u001b[32m(\u001b[0m\u001b[32m3\u001b[0m\u001b[32m)\u001b[0m\u001b[32m vs. Mavericks \u001b[0m\u001b[32m(\u001b[0m\u001b[32m5\u001b[0m\u001b[32m)\u001b[0m\u001b[32m - NBA.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.nba.com/playoffs/2024/west-final\\\\\", \\\\\"content\\\\\": \\\\\"The Dallas Mavericks and Minnesota Timberwolves have advanced to the 2024 Western Conference Finals during the NBA playoffs.\\\\\", \\\\\"score\\\\\": 0.8884594, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"NBA Conference Finals Schedule: Full List of Games & Results\\\\\", \\\\\"url\\\\\": \\\\\"https://www.si.com/nba/nba-conference-finals-schedule-full-list-of-games-results\\\\\", \\\\\"content\\\\\": \\\\\"The 2024 NBA conference finals matchups are set. Here\\'s the schedule for all the games. ... Western Conference First Round \u001b[0m\u001b[32m(\u001b[0m\u001b[32m1\u001b[0m\u001b[32m)\u001b[0m\u001b[32m Oklahoma City Thunder def. \u001b[0m\u001b[32m(\u001b[0m\u001b[32m8\u001b[0m\u001b[32m)\u001b[0m\u001b[32m New Orleans Pelicans in 4 games\\\\\", \\\\\"score\\\\\": 0.85008353, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"NBA Finals 2024 - Celtics-Mavericks news, schedule, scores and ... - ESPN\\\\\", \\\\\"url\\\\\": \\\\\"https://www.espn.com/nba/story/_/id/39943302/nba-playoffs-2024-conference-finals-news-scores-highlights\\\\\", \\\\\"content\\\\\": \\\\\"The Boston Celtics are the 2024 NBA Champions. ... Western Conference. Final 2023-24 NBA regular-season standings. Which team left standing has the most trips to the NBA Finals? Here is a look at\\\\\", \\\\\"score\\\\\": 0.81979275, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[1;39m{\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'input'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;39m[\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"system\",\"content\":\"You are a helpful assistant. Use search tool to answer the questions. \"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"call_id\":\"b7d9e0dd-4d6d-47db-9d81-3d7834f6e53d\",\"tool_name\":\"brave_search\",\"arguments\":\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"query\":\"NBA Western Conference Finals 2024 teams\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"ipython\",\"call_id\":\"b7d9e0dd-4d6d-47db-9d81-3d7834f6e53d\",\"tool_name\":\"brave_search\",\"content\":\"\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"query\\\\\": \\\\\"NBA Western Conference Finals 2024 teams\\\\\", \\\\\"top_k\\\\\": \u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference Finals - Basketball-Reference.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.basketball-reference.com/playoffs/2024-nba-western-conference-finals-mavericks-vs-timberwolves.html\\\\\", \\\\\"content\\\\\": \\\\\"2024 NBA Western Conference Finals Mavericks vs. Timberwolves League Champion: Boston Celtics. Finals MVP: Jaylen Brown \u001b[0m\u001b[32m(\u001b[0m\u001b[32m20.8 / 5.4 / 5.0\u001b[0m\u001b[32m)\u001b[0m\u001b[32m 2024 Playoff Leaders: PTS: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m635\u001b[0m\u001b[32m)\u001b[0m\u001b[32m TRB: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m208\u001b[0m\u001b[32m)\u001b[0m\u001b[32m AST: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m178\u001b[0m\u001b[32m)\u001b[0m\u001b[32m WS: Derrick White \u001b[0m\u001b[32m(\u001b[0m\u001b[32m2.9\u001b[0m\u001b[32m)\u001b[0m\u001b[32m More playoffs info\\\\\", \\\\\"score\\\\\": 0.9310187, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates, schedule and more - Sportskeeda\\\\\", \\\\\"url\\\\\": \\\\\"https://www.sportskeeda.com/basketball/news-nba-western-conference-finals-2024-dates-schedule-and-more\\\\\", \\\\\"content\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates & Schedule The 2023-24 NBA Western Conference Finals will start on Wednesday, May 22. The Mavericks will face the team that wins in Game 7 between the\\\\\", \\\\\"score\\\\\": 0.8914433, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"2024 Playoffs: West Finals | Timberwolves \u001b[0m\u001b[32m(\u001b[0m\u001b[32m3\u001b[0m\u001b[32m)\u001b[0m\u001b[32m vs. Mavericks \u001b[0m\u001b[32m(\u001b[0m\u001b[32m5\u001b[0m\u001b[32m)\u001b[0m\u001b[32m - NBA.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.nba.com/playoffs/2024/west-final\\\\\", \\\\\"content\\\\\": \\\\\"The Dallas Mavericks and Minnesota Timberwolves have advanced to the 2024 Western Conference Finals during the NBA playoffs.\\\\\", \\\\\"score\\\\\": 0.8884594, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"NBA Conference Finals Schedule: Full List of Games & Results\\\\\", \\\\\"url\\\\\": \\\\\"https://www.si.com/nba/nba-conference-finals-schedule-full-list-of-games-results\\\\\", \\\\\"content\\\\\": \\\\\"The 2024 NBA conference finals matchups are set. Here\\'s the schedule for all the games. ... Western Conference First Round \u001b[0m\u001b[32m(\u001b[0m\u001b[32m1\u001b[0m\u001b[32m)\u001b[0m\u001b[32m Oklahoma City Thunder def. \u001b[0m\u001b[32m(\u001b[0m\u001b[32m8\u001b[0m\u001b[32m)\u001b[0m\u001b[32m New Orleans Pelicans in 4 games\\\\\", \\\\\"score\\\\\": 0.85008353, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"NBA Finals 2024 - Celtics-Mavericks news, schedule, scores and ... - ESPN\\\\\", \\\\\"url\\\\\": \\\\\"https://www.espn.com/nba/story/_/id/39943302/nba-playoffs-2024-conference-finals-news-scores-highlights\\\\\", \\\\\"content\\\\\": \\\\\"The Boston Celtics are the 2024 NBA Champions. ... Western Conference. Final 2023-24 NBA regular-season standings. Which team left standing has the most trips to the NBA Finals? Here is a look at\\\\\", \\\\\"score\\\\\": 0.81979275, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[1;39m]\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'output'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'content: The teams that played in the NBA Western Conference Finals of 2024 were the Dallas Mavericks and the Minnesota Timberwolves. tool_calls: \u001b[0m\u001b[32m[\u001b[0m\u001b[32m]\u001b[0m\u001b[32m'\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[1;39m{\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'input'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;39m[\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"system\",\"content\":\"You are a helpful assistant. Use search tool to answer the questions. \"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"call_id\":\"b7d9e0dd-4d6d-47db-9d81-3d7834f6e53d\",\"tool_name\":\"brave_search\",\"arguments\":\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"query\":\"NBA Western Conference Finals 2024 teams\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"ipython\",\"call_id\":\"b7d9e0dd-4d6d-47db-9d81-3d7834f6e53d\",\"tool_name\":\"brave_search\",\"content\":\"\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"query\\\\\": \\\\\"NBA Western Conference Finals 2024 teams\\\\\", \\\\\"top_k\\\\\": \u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference Finals - Basketball-Reference.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.basketball-reference.com/playoffs/2024-nba-western-conference-finals-mavericks-vs-timberwolves.html\\\\\", \\\\\"content\\\\\": \\\\\"2024 NBA Western Conference Finals Mavericks vs. Timberwolves League Champion: Boston Celtics. Finals MVP: Jaylen Brown \u001b[0m\u001b[32m(\u001b[0m\u001b[32m20.8 / 5.4 / 5.0\u001b[0m\u001b[32m)\u001b[0m\u001b[32m 2024 Playoff Leaders: PTS: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m635\u001b[0m\u001b[32m)\u001b[0m\u001b[32m TRB: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m208\u001b[0m\u001b[32m)\u001b[0m\u001b[32m AST: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m178\u001b[0m\u001b[32m)\u001b[0m\u001b[32m WS: Derrick White \u001b[0m\u001b[32m(\u001b[0m\u001b[32m2.9\u001b[0m\u001b[32m)\u001b[0m\u001b[32m More playoffs info\\\\\", \\\\\"score\\\\\": 0.9310187, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates, schedule and more - Sportskeeda\\\\\", \\\\\"url\\\\\": \\\\\"https://www.sportskeeda.com/basketball/news-nba-western-conference-finals-2024-dates-schedule-and-more\\\\\", \\\\\"content\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates & Schedule The 2023-24 NBA Western Conference Finals will start on Wednesday, May 22. The Mavericks will face the team that wins in Game 7 between the\\\\\", \\\\\"score\\\\\": 0.8914433, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"2024 Playoffs: West Finals | Timberwolves \u001b[0m\u001b[32m(\u001b[0m\u001b[32m3\u001b[0m\u001b[32m)\u001b[0m\u001b[32m vs. Mavericks \u001b[0m\u001b[32m(\u001b[0m\u001b[32m5\u001b[0m\u001b[32m)\u001b[0m\u001b[32m - NBA.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.nba.com/playoffs/2024/west-final\\\\\", \\\\\"content\\\\\": \\\\\"The Dallas Mavericks and Minnesota Timberwolves have advanced to the 2024 Western Conference Finals during the NBA playoffs.\\\\\", \\\\\"score\\\\\": 0.8884594, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"NBA Conference Finals Schedule: Full List of Games & Results\\\\\", \\\\\"url\\\\\": \\\\\"https://www.si.com/nba/nba-conference-finals-schedule-full-list-of-games-results\\\\\", \\\\\"content\\\\\": \\\\\"The 2024 NBA conference finals matchups are set. Here\\'s the schedule for all the games. ... Western Conference First Round \u001b[0m\u001b[32m(\u001b[0m\u001b[32m1\u001b[0m\u001b[32m)\u001b[0m\u001b[32m Oklahoma City Thunder def. \u001b[0m\u001b[32m(\u001b[0m\u001b[32m8\u001b[0m\u001b[32m)\u001b[0m\u001b[32m New Orleans Pelicans in 4 games\\\\\", \\\\\"score\\\\\": 0.85008353, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"NBA Finals 2024 - Celtics-Mavericks news, schedule, scores and ... - ESPN\\\\\", \\\\\"url\\\\\": \\\\\"https://www.espn.com/nba/story/_/id/39943302/nba-playoffs-2024-conference-finals-news-scores-highlights\\\\\", \\\\\"content\\\\\": \\\\\"The Boston Celtics are the 2024 NBA Champions. ... Western Conference. Final 2023-24 NBA regular-season standings. Which team left standing has the most trips to the NBA Finals? Here is a look at\\\\\", \\\\\"score\\\\\": 0.81979275, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"The teams that played in the NBA Western Conference Finals of 2024 were the Dallas Mavericks and the Minnesota Timberwolves.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"user\",\"content\":\"In which episode and season of South Park does Bill Cosby \u001b[0m\u001b[32m(\u001b[0m\u001b[32mBSM-471\u001b[0m\u001b[32m)\u001b[0m\u001b[32m first appear? Give me the number and title.\",\"context\":null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[1;39m]\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'output'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m\"content: tool_calls: \u001b[0m\u001b[32m[\u001b[0m\u001b[32mToolCall\u001b[0m\u001b[32m(\u001b[0m\u001b[32mcall_id\u001b[0m\u001b[32m='1e487e8e-a15f-4137-854a-1d4979a70b8c', \u001b[0m\u001b[32mtool_name\u001b[0m\u001b[32m=\u001b[0m\u001b[32m, \u001b[0m\u001b[32marguments\u001b[0m\u001b[32m=\u001b[0m\u001b[32m{\u001b[0m\u001b[32m'query': 'Bill Cosby South Park episode'\u001b[0m\u001b[32m}\u001b[0m\u001b[32m)\u001b[0m\u001b[32m]\u001b[0m\u001b[32m\"\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[1m}\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[1m{\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'input'\u001b[0m: \u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"call_id\":\"1e487e8e-a15f-4137-854a-1d4979a70b8c\",\"tool_name\":\"brave_search\",\"arguments\":\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"query\":\"Bill Cosby South Park episode\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'output'\u001b[0m: \u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"ipython\",\"call_id\":\"1e487e8e-a15f-4137-854a-1d4979a70b8c\",\"tool_name\":\"brave_search\",\"content\":\"\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"query\\\\\": \\\\\"Bill Cosby South Park episode\\\\\", \\\\\"top_k\\\\\": \u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Bill Cosby | South Park Archives | Fandom\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.fandom.com/wiki/Bill_Cosby\\\\\", \\\\\"content\\\\\": \\\\\"For other uses, see Bill \u001b[0m\u001b[32m(\u001b[0m\u001b[32mDisambiguation\u001b[0m\u001b[32m)\u001b[0m\u001b[32m. William Henry \\\\\\\\\\\\\"Bill\\\\\\\\\\\\\" Cosby Jr. African-American comedian, actor, and serial rapist. He first appears in the Season Five episode, \\\\\\\\\\\\\"Here Comes the Neighborhood\\\\\\\\\\\\\", as one of the wealthy African-Americans who move to South Park. He returned as a hologram in the Season Eighteen episode, \\\\\\\\\\\\\"#HappyHolograms\\\\\\\\\\\\\" where he is shown trying to molest pop star Taylor\\\\\", \\\\\"score\\\\\": 0.82288796, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Trapper Keeper \u001b[0m\u001b[32m(\u001b[0m\u001b[32mSouth Park\u001b[0m\u001b[32m)\u001b[0m\u001b[32m - Wikipedia\\\\\", \\\\\"url\\\\\": \\\\\"https://en.wikipedia.org/wiki/Trapper_Keeper_\u001b[0m\u001b[32m(\u001b[0m\u001b[32mSouth_Park\u001b[0m\u001b[32m)\u001b[0m\u001b[32m\\\\\", \\\\\"content\\\\\": \\\\\"Bill Cosby warns that if the Trapper Keeper assimilates with the supercomputer at Cheyenne Mountain, it will become unstoppable. ... It is one of the many South Park episodes that parodies a current event. \u001b[0m\u001b[32m[\u001b[0m\u001b[32m1\u001b[0m\u001b[32m]\u001b[0m\u001b[32m The main plot of the episode involving the Trapper Keeper was written before the election, \u001b[0m\u001b[32m[\u001b[0m\u001b[32m1\u001b[0m\u001b[32m]\u001b[0m\u001b[32m\\\\\", \\\\\"score\\\\\": 0.75659186, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Bill Cosby is Here to See You - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/video-clips/wfot8s/south-park-bill-cosby-is-here-to-see-you\\\\\", \\\\\"content\\\\\": \\\\\"Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. ... South Park. Bill Cosby is Here to See You. Season 18 E 10 \\\\\\\\u2022 12/10/2014. Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. More. Watch Random Episode. Watching. 01:11. Please Welcome \\\\\\\\\\\\\"Cartman Bra\\\\\\\\\\\\\" South Park S18 E9.\\\\\", \\\\\"score\\\\\": 0.7156829, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Bill Cosby and Taylor Swift Duet - South Park Studios\\\\\", \\\\\"url\\\\\": \\\\\"https://www.southparkstudios.com/video-clips/90r7i1/south-park-bill-cosby-and-taylor-swift-duet\\\\\", \\\\\"content\\\\\": \\\\\"The holiday special continues with Bill Cosby and Taylor Swift\\'s rendition of \\\\\\\\\\\\\"It\\'s Snowing Out There\\\\\\\\\\\\\". ... Full Episodes. Collections. Random Episode. Full Episodes. Events. Wiki. News. Avatar. Shop. Forum. Games. South Park. Menu. Episodes & Videos. About. South Park. Bill Cosby and Taylor Swift Duet. Season 18 E 10 \\\\\\\\u2022 12/10/2014. The\\\\\", \\\\\"score\\\\\": 0.64639384, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Bill Cosby \u001b[0m\u001b[32m(\u001b[0m\u001b[32mandroid\u001b[0m\u001b[32m)\u001b[0m\u001b[32m | South Park Character ... - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/wiki/Bill_Cosby_\u001b[0m\u001b[32m(\u001b[0m\u001b[32mandroid\u001b[0m\u001b[32m)\u001b[0m\u001b[32m\\\\\", \\\\\"content\\\\\": \\\\\"About. Sent back in time to destroy Eric Cartman\\'s Dawson\\'s Creek Trapper Keeper before it manifests into an omnipotent supercomputer that can destroy all humanity, \\\\\\\\\\\\\"Bill Cosby\\\\\\\\\\\\\" is really VSM471, an android or cyborg of some kind engineered by \\'hoomans\\' in the distant future. He fails in his initial missions to infiltrate South Park Elementary\\'s 4th Grade class, destroy the Trapper Keeper or\\\\\", \\\\\"score\\\\\": 0.56460327, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[1m}\u001b[0m,\n", - "\u001b[2;32m│ \u001b[0m\u001b[1m{\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'input'\u001b[0m: \u001b[1m[\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"system\",\"content\":\"You are a helpful assistant. Use search tool to answer the questions. \"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"call_id\":\"b7d9e0dd-4d6d-47db-9d81-3d7834f6e53d\",\"tool_name\":\"brave_search\",\"arguments\":\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"query\":\"NBA Western Conference Finals 2024 teams\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"ipython\",\"call_id\":\"b7d9e0dd-4d6d-47db-9d81-3d7834f6e53d\",\"tool_name\":\"brave_search\",\"content\":\"\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"query\\\\\": \\\\\"NBA Western Conference Finals 2024 teams\\\\\", \\\\\"top_k\\\\\": \u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference Finals - Basketball-Reference.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.basketball-reference.com/playoffs/2024-nba-western-conference-finals-mavericks-vs-timberwolves.html\\\\\", \\\\\"content\\\\\": \\\\\"2024 NBA Western Conference Finals Mavericks vs. Timberwolves League Champion: Boston Celtics. Finals MVP: Jaylen Brown \u001b[0m\u001b[32m(\u001b[0m\u001b[32m20.8 / 5.4 / 5.0\u001b[0m\u001b[32m)\u001b[0m\u001b[32m 2024 Playoff Leaders: PTS: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m635\u001b[0m\u001b[32m)\u001b[0m\u001b[32m TRB: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m208\u001b[0m\u001b[32m)\u001b[0m\u001b[32m AST: Luka Don\\\\\\\\u010di\\\\\\\\u0107 \u001b[0m\u001b[32m(\u001b[0m\u001b[32m178\u001b[0m\u001b[32m)\u001b[0m\u001b[32m WS: Derrick White \u001b[0m\u001b[32m(\u001b[0m\u001b[32m2.9\u001b[0m\u001b[32m)\u001b[0m\u001b[32m More playoffs info\\\\\", \\\\\"score\\\\\": 0.9310187, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates, schedule and more - Sportskeeda\\\\\", \\\\\"url\\\\\": \\\\\"https://www.sportskeeda.com/basketball/news-nba-western-conference-finals-2024-dates-schedule-and-more\\\\\", \\\\\"content\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates & Schedule The 2023-24 NBA Western Conference Finals will start on Wednesday, May 22. The Mavericks will face the team that wins in Game 7 between the\\\\\", \\\\\"score\\\\\": 0.8914433, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"2024 Playoffs: West Finals | Timberwolves \u001b[0m\u001b[32m(\u001b[0m\u001b[32m3\u001b[0m\u001b[32m)\u001b[0m\u001b[32m vs. Mavericks \u001b[0m\u001b[32m(\u001b[0m\u001b[32m5\u001b[0m\u001b[32m)\u001b[0m\u001b[32m - NBA.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.nba.com/playoffs/2024/west-final\\\\\", \\\\\"content\\\\\": \\\\\"The Dallas Mavericks and Minnesota Timberwolves have advanced to the 2024 Western Conference Finals during the NBA playoffs.\\\\\", \\\\\"score\\\\\": 0.8884594, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"NBA Conference Finals Schedule: Full List of Games & Results\\\\\", \\\\\"url\\\\\": \\\\\"https://www.si.com/nba/nba-conference-finals-schedule-full-list-of-games-results\\\\\", \\\\\"content\\\\\": \\\\\"The 2024 NBA conference finals matchups are set. Here\\'s the schedule for all the games. ... Western Conference First Round \u001b[0m\u001b[32m(\u001b[0m\u001b[32m1\u001b[0m\u001b[32m)\u001b[0m\u001b[32m Oklahoma City Thunder def. \u001b[0m\u001b[32m(\u001b[0m\u001b[32m8\u001b[0m\u001b[32m)\u001b[0m\u001b[32m New Orleans Pelicans in 4 games\\\\\", \\\\\"score\\\\\": 0.85008353, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"NBA Finals 2024 - Celtics-Mavericks news, schedule, scores and ... - ESPN\\\\\", \\\\\"url\\\\\": \\\\\"https://www.espn.com/nba/story/_/id/39943302/nba-playoffs-2024-conference-finals-news-scores-highlights\\\\\", \\\\\"content\\\\\": \\\\\"The Boston Celtics are the 2024 NBA Champions. ... Western Conference. Final 2023-24 NBA regular-season standings. Which team left standing has the most trips to the NBA Finals? Here is a look at\\\\\", \\\\\"score\\\\\": 0.81979275, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"The teams that played in the NBA Western Conference Finals of 2024 were the Dallas Mavericks and the Minnesota Timberwolves.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"user\",\"content\":\"In which episode and season of South Park does Bill Cosby \u001b[0m\u001b[32m(\u001b[0m\u001b[32mBSM-471\u001b[0m\u001b[32m)\u001b[0m\u001b[32m first appear? Give me the number and title.\",\"context\":null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":\u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"call_id\":\"1e487e8e-a15f-4137-854a-1d4979a70b8c\",\"tool_name\":\"brave_search\",\"arguments\":\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"query\":\"Bill Cosby South Park episode\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"ipython\",\"call_id\":\"1e487e8e-a15f-4137-854a-1d4979a70b8c\",\"tool_name\":\"brave_search\",\"content\":\"\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"query\\\\\": \\\\\"Bill Cosby South Park episode\\\\\", \\\\\"top_k\\\\\": \u001b[0m\u001b[32m[\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Bill Cosby | South Park Archives | Fandom\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.fandom.com/wiki/Bill_Cosby\\\\\", \\\\\"content\\\\\": \\\\\"For other uses, see Bill \u001b[0m\u001b[32m(\u001b[0m\u001b[32mDisambiguation\u001b[0m\u001b[32m)\u001b[0m\u001b[32m. William Henry \\\\\\\\\\\\\"Bill\\\\\\\\\\\\\" Cosby Jr. African-American comedian, actor, and serial rapist. He first appears in the Season Five episode, \\\\\\\\\\\\\"Here Comes the Neighborhood\\\\\\\\\\\\\", as one of the wealthy African-Americans who move to South Park. He returned as a hologram in the Season Eighteen episode, \\\\\\\\\\\\\"#HappyHolograms\\\\\\\\\\\\\" where he is shown trying to molest pop star Taylor\\\\\", \\\\\"score\\\\\": 0.82288796, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Trapper Keeper \u001b[0m\u001b[32m(\u001b[0m\u001b[32mSouth Park\u001b[0m\u001b[32m)\u001b[0m\u001b[32m - Wikipedia\\\\\", \\\\\"url\\\\\": \\\\\"https://en.wikipedia.org/wiki/Trapper_Keeper_\u001b[0m\u001b[32m(\u001b[0m\u001b[32mSouth_Park\u001b[0m\u001b[32m)\u001b[0m\u001b[32m\\\\\", \\\\\"content\\\\\": \\\\\"Bill Cosby warns that if the Trapper Keeper assimilates with the supercomputer at Cheyenne Mountain, it will become unstoppable. ... It is one of the many South Park episodes that parodies a current event. \u001b[0m\u001b[32m[\u001b[0m\u001b[32m1\u001b[0m\u001b[32m]\u001b[0m\u001b[32m The main plot of the episode involving the Trapper Keeper was written before the election, \u001b[0m\u001b[32m[\u001b[0m\u001b[32m1\u001b[0m\u001b[32m]\u001b[0m\u001b[32m\\\\\", \\\\\"score\\\\\": 0.75659186, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Bill Cosby is Here to See You - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/video-clips/wfot8s/south-park-bill-cosby-is-here-to-see-you\\\\\", \\\\\"content\\\\\": \\\\\"Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. ... South Park. Bill Cosby is Here to See You. Season 18 E 10 \\\\\\\\u2022 12/10/2014. Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. More. Watch Random Episode. Watching. 01:11. Please Welcome \\\\\\\\\\\\\"Cartman Bra\\\\\\\\\\\\\" South Park S18 E9.\\\\\", \\\\\"score\\\\\": 0.7156829, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Bill Cosby and Taylor Swift Duet - South Park Studios\\\\\", \\\\\"url\\\\\": \\\\\"https://www.southparkstudios.com/video-clips/90r7i1/south-park-bill-cosby-and-taylor-swift-duet\\\\\", \\\\\"content\\\\\": \\\\\"The holiday special continues with Bill Cosby and Taylor Swift\\'s rendition of \\\\\\\\\\\\\"It\\'s Snowing Out There\\\\\\\\\\\\\". ... Full Episodes. Collections. Random Episode. Full Episodes. Events. Wiki. News. Avatar. Shop. Forum. Games. South Park. Menu. Episodes & Videos. About. South Park. Bill Cosby and Taylor Swift Duet. Season 18 E 10 \\\\\\\\u2022 12/10/2014. The\\\\\", \\\\\"score\\\\\": 0.64639384, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m, \u001b[0m\u001b[32m{\u001b[0m\u001b[32m\\\\\"title\\\\\": \\\\\"Bill Cosby \u001b[0m\u001b[32m(\u001b[0m\u001b[32mandroid\u001b[0m\u001b[32m)\u001b[0m\u001b[32m | South Park Character ... - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/wiki/Bill_Cosby_\u001b[0m\u001b[32m(\u001b[0m\u001b[32mandroid\u001b[0m\u001b[32m)\u001b[0m\u001b[32m\\\\\", \\\\\"content\\\\\": \\\\\"About. Sent back in time to destroy Eric Cartman\\'s Dawson\\'s Creek Trapper Keeper before it manifests into an omnipotent supercomputer that can destroy all humanity, \\\\\\\\\\\\\"Bill Cosby\\\\\\\\\\\\\" is really VSM471, an android or cyborg of some kind engineered by \\'hoomans\\' in the distant future. He fails in his initial missions to infiltrate South Park Elementary\\'s 4th Grade class, destroy the Trapper Keeper or\\\\\", \\\\\"score\\\\\": 0.56460327, \\\\\"raw_content\\\\\": null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m\"\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[1m]\u001b[0m,\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'output'\u001b[0m: \u001b[32m'content: Bill Cosby \u001b[0m\u001b[32m(\u001b[0m\u001b[32mBSM-471\u001b[0m\u001b[32m)\u001b[0m\u001b[32m first appears in the Season 4 episode \"Trapper Keeper\" of South Park. tool_calls: \u001b[0m\u001b[32m[\u001b[0m\u001b[32m]\u001b[0m\u001b[32m'\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[1m}\u001b[0m\n", - "\u001b[1m]\u001b[0m\n" - ], - "text/html": [ - "
[\n",
-              "│   {\n",
-              "│   │   'input': [\n",
-              "│   │   │   '{\"role\":\"system\",\"content\":\"You are a helpful assistant. Use search tool to answer the questions. \"}',\n",
-              "│   │   │   '{\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null}',\n",
-              "│   │   │   '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"b7d9e0dd-4d6d-47db-9d81-3d7834f6e53d\",\"tool_name\":\"brave_search\",\"arguments\":{\"query\":\"NBA Western Conference Finals 2024 teams\"}}]}',\n",
-              "│   │   │   '{\"role\":\"ipython\",\"call_id\":\"b7d9e0dd-4d6d-47db-9d81-3d7834f6e53d\",\"tool_name\":\"brave_search\",\"content\":\"{\\\\\"query\\\\\": \\\\\"NBA Western Conference Finals 2024 teams\\\\\", \\\\\"top_k\\\\\": [{\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference Finals - Basketball-Reference.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.basketball-reference.com/playoffs/2024-nba-western-conference-finals-mavericks-vs-timberwolves.html\\\\\", \\\\\"content\\\\\": \\\\\"2024 NBA Western Conference Finals Mavericks vs. Timberwolves League Champion: Boston Celtics. Finals MVP: Jaylen Brown (20.8 / 5.4 / 5.0) 2024 Playoff Leaders: PTS: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (635) TRB: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (208) AST: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (178) WS: Derrick White (2.9) More playoffs info\\\\\", \\\\\"score\\\\\": 0.9310187, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates, schedule and more - Sportskeeda\\\\\", \\\\\"url\\\\\": \\\\\"https://www.sportskeeda.com/basketball/news-nba-western-conference-finals-2024-dates-schedule-and-more\\\\\", \\\\\"content\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates & Schedule The 2023-24 NBA Western Conference Finals will start on Wednesday, May 22. The Mavericks will face the team that wins in Game 7 between the\\\\\", \\\\\"score\\\\\": 0.8914433, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"2024 Playoffs: West Finals | Timberwolves (3) vs. Mavericks (5) - NBA.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.nba.com/playoffs/2024/west-final\\\\\", \\\\\"content\\\\\": \\\\\"The Dallas Mavericks and Minnesota Timberwolves have advanced to the 2024 Western Conference Finals during the NBA playoffs.\\\\\", \\\\\"score\\\\\": 0.8884594, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"NBA Conference Finals Schedule: Full List of Games & Results\\\\\", \\\\\"url\\\\\": \\\\\"https://www.si.com/nba/nba-conference-finals-schedule-full-list-of-games-results\\\\\", \\\\\"content\\\\\": \\\\\"The 2024 NBA conference finals matchups are set. Here\\'s the schedule for all the games. ... Western Conference First Round (1) Oklahoma City Thunder def. (8) New Orleans Pelicans in 4 games\\\\\", \\\\\"score\\\\\": 0.85008353, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"NBA Finals 2024 - Celtics-Mavericks news, schedule, scores and ... - ESPN\\\\\", \\\\\"url\\\\\": \\\\\"https://www.espn.com/nba/story/_/id/39943302/nba-playoffs-2024-conference-finals-news-scores-highlights\\\\\", \\\\\"content\\\\\": \\\\\"The Boston Celtics are the 2024 NBA Champions. ... Western Conference. Final 2023-24 NBA regular-season standings. Which team left standing has the most trips to the NBA Finals? Here is a look at\\\\\", \\\\\"score\\\\\": 0.81979275, \\\\\"raw_content\\\\\": null}]}\"}',\n",
-              "│   │   │   '{\"role\":\"assistant\",\"content\":\"The teams that played in the NBA Western Conference Finals of 2024 were the Dallas Mavericks and the Minnesota Timberwolves.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[]}',\n",
-              "│   │   │   '{\"role\":\"user\",\"content\":\"In which episode and season of South Park does Bill Cosby (BSM-471) first appear? Give me the number and title.\",\"context\":null}',\n",
-              "│   │   │   '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"1e487e8e-a15f-4137-854a-1d4979a70b8c\",\"tool_name\":\"brave_search\",\"arguments\":{\"query\":\"Bill Cosby South Park episode\"}}]}',\n",
-              "│   │   │   '{\"role\":\"ipython\",\"call_id\":\"1e487e8e-a15f-4137-854a-1d4979a70b8c\",\"tool_name\":\"brave_search\",\"content\":\"{\\\\\"query\\\\\": \\\\\"Bill Cosby South Park episode\\\\\", \\\\\"top_k\\\\\": [{\\\\\"title\\\\\": \\\\\"Bill Cosby | South Park Archives | Fandom\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.fandom.com/wiki/Bill_Cosby\\\\\", \\\\\"content\\\\\": \\\\\"For other uses, see Bill (Disambiguation). William Henry \\\\\\\\\\\\\"Bill\\\\\\\\\\\\\" Cosby Jr. African-American comedian, actor, and serial rapist. He first appears in the Season Five episode, \\\\\\\\\\\\\"Here Comes the Neighborhood\\\\\\\\\\\\\", as one of the wealthy African-Americans who move to South Park. He returned as a hologram in the Season Eighteen episode, \\\\\\\\\\\\\"#HappyHolograms\\\\\\\\\\\\\" where he is shown trying to molest pop star Taylor\\\\\", \\\\\"score\\\\\": 0.82288796, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Trapper Keeper (South Park) - Wikipedia\\\\\", \\\\\"url\\\\\": \\\\\"https://en.wikipedia.org/wiki/Trapper_Keeper_(South_Park)\\\\\", \\\\\"content\\\\\": \\\\\"Bill Cosby warns that if the Trapper Keeper assimilates with the supercomputer at Cheyenne Mountain, it will become unstoppable. ... It is one of the many South Park episodes that parodies a current event. [1] The main plot of the episode involving the Trapper Keeper was written before the election, [1]\\\\\", \\\\\"score\\\\\": 0.75659186, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Bill Cosby is Here to See You - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/video-clips/wfot8s/south-park-bill-cosby-is-here-to-see-you\\\\\", \\\\\"content\\\\\": \\\\\"Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. ... South Park. Bill Cosby is Here to See You. Season 18 E 10 \\\\\\\\u2022 12/10/2014. Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. More. Watch Random Episode. Watching. 01:11. Please Welcome \\\\\\\\\\\\\"Cartman Bra\\\\\\\\\\\\\" South Park S18 E9.\\\\\", \\\\\"score\\\\\": 0.7156829, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Bill Cosby and Taylor Swift Duet - South Park Studios\\\\\", \\\\\"url\\\\\": \\\\\"https://www.southparkstudios.com/video-clips/90r7i1/south-park-bill-cosby-and-taylor-swift-duet\\\\\", \\\\\"content\\\\\": \\\\\"The holiday special continues with Bill Cosby and Taylor Swift\\'s rendition of \\\\\\\\\\\\\"It\\'s Snowing Out There\\\\\\\\\\\\\". ... Full Episodes. Collections. Random Episode. Full Episodes. Events. Wiki. News. Avatar. Shop. Forum. Games. South Park. Menu. Episodes & Videos. About. South Park. Bill Cosby and Taylor Swift Duet. Season 18 E 10 \\\\\\\\u2022 12/10/2014. The\\\\\", \\\\\"score\\\\\": 0.64639384, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Bill Cosby (android) | South Park Character ... - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/wiki/Bill_Cosby_(android)\\\\\", \\\\\"content\\\\\": \\\\\"About. Sent back in time to destroy Eric Cartman\\'s Dawson\\'s Creek Trapper Keeper before it manifests into an omnipotent supercomputer that can destroy all humanity, \\\\\\\\\\\\\"Bill Cosby\\\\\\\\\\\\\" is really VSM471, an android or cyborg of some kind engineered by \\'hoomans\\' in the distant future. He fails in his initial missions to infiltrate South Park Elementary\\'s 4th Grade class, destroy the Trapper Keeper or\\\\\", \\\\\"score\\\\\": 0.56460327, \\\\\"raw_content\\\\\": null}]}\"}',\n",
-              "│   │   │   '{\"role\":\"assistant\",\"content\":\"Bill Cosby (BSM-471) first appears in the Season 4 episode \\\\\"Trapper Keeper\\\\\" of South Park.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[]}',\n",
-              "│   │   │   '{\"role\":\"user\",\"content\":\"What is the British-American kickboxer Andrew Tate\\'s kickboxing name?\",\"context\":null}'\n",
-              "│   │   ],\n",
-              "│   │   'output': \"content:  tool_calls: [ToolCall(call_id='44705eaf-b371-4841-b0ee-5eb21a5d7f36', tool_name=<BuiltinTool.brave_search: 'brave_search'>, arguments={'query': 'Andrew Tate kickboxing name'})]\"\n",
-              "│   },\n",
-              "│   {\n",
-              "│   │   'input': '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"44705eaf-b371-4841-b0ee-5eb21a5d7f36\",\"tool_name\":\"brave_search\",\"arguments\":{\"query\":\"Andrew Tate kickboxing name\"}}]}',\n",
-              "│   │   'output': '{\"role\":\"ipython\",\"call_id\":\"44705eaf-b371-4841-b0ee-5eb21a5d7f36\",\"tool_name\":\"brave_search\",\"content\":\"{\\\\\"query\\\\\": \\\\\"Andrew Tate kickboxing name\\\\\", \\\\\"top_k\\\\\": [{\\\\\"title\\\\\": \\\\\"50 Facts About Andrew Tate - Facts.net\\\\\", \\\\\"url\\\\\": \\\\\"https://facts.net/andrew-tate-facts/\\\\\", \\\\\"content\\\\\": \\\\\"Full Name: Andrew Tate\\'s full name is Emory Andrew Tate III, named after his father, a celebrated chess player. Date of Birth: ... Kickboxing Start: Tate began his kickboxing career in 2005, starting his journey as a professional fighter, which would later be a significant part of his persona. First Championship:\\\\\", \\\\\"score\\\\\": 0.8967681, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"The Life Of Andrew Tate (By Andrew Tate Himself)\\\\\", \\\\\"url\\\\\": \\\\\"https://sidekickboxing.co.uk/the-life-of-andrew-king-cobra-tate/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate stats. Fight Name: Cobra Tate. Born: 1 December 1986. Weight: 90 KG. Weight Class: Cruiserweight. Height: 1.92m. Fight Record: Wins - 76, Losses - 9. ... Andrew Tate\\'s Kickboxing Career. Andrew Tate has always fought credible opponents right from the beginning of his kickboxing career. One of his first professional fights on\\\\\", \\\\\"score\\\\\": 0.8795718, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"About Andrew Tate | The Real World\\\\\", \\\\\"url\\\\\": \\\\\"https://www.taterealworldofficial.com/about-andrew-tate\\\\\", \\\\\"content\\\\\": \\\\\"Emory Andrew Tate III (born December 14, 1986) is an American-British kickboxer from Chicago, Illinois, who competes in the cruiserweight and heavyweight divisions. ... Tate challenged Paul Randall for the vacant ISKA English Kickboxing Light-cruiserweight title. Tate won his first ISKA Kickboxing title stopping Randall in the fifth round of\\\\\", \\\\\"score\\\\\": 0.8386933, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Andrew Tate - Fight Record - Muay Thai Records\\\\\", \\\\\"url\\\\\": \\\\\"https://muaythairecords.com/fighters/andrew-tate\\\\\", \\\\\"content\\\\\": \\\\\"Andrew \\\\\\\\\\\\\"King Cobra\\\\\\\\\\\\\" Tate is a 38-year-old Muay Thai fighter. With a record of 23-8-0, including 32 knockouts, standing at 6\\\\\\\\u2032 4\\\\\\\\u2033 and weighing 198 lbs. Originally from Luton, United Kingdom. ... WIN Dec -Kickboxing Jean Luc Beno\\\\\\\\u00eet. 14th Mar 2015 -Boxe in D\\\\\\\\u00e9fi 16. Andrew Tate defeated Jean Luc Beno\\\\\\\\u00eet by decision. ... Name: Andrew Tate\\\\\", \\\\\"score\\\\\": 0.8194462, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Andrew Tate: Kickboxing Record, Facts, Height, Weight, Age, Biography\\\\\", \\\\\"url\\\\\": \\\\\"https://www.lowkickmma.com/andrew-tate-kickboxing-record-facts-height-weight-age-biography/\\\\\", \\\\\"content\\\\\": \\\\\"Birth Name: Emory Andrew Tate III: Date of Birth: 1 December 1986: Place of Birth: Washington, D.C., U.S. ... In his professional kickboxing career, Andrew Tate won 32 of his fights by knockout.\\\\\", \\\\\"score\\\\\": 0.7992077, \\\\\"raw_content\\\\\": null}]}\"}'\n",
-              "│   },\n",
-              "│   {\n",
-              "│   │   'input': [\n",
-              "│   │   │   '{\"role\":\"system\",\"content\":\"You are a helpful assistant. Use search tool to answer the questions. \"}',\n",
-              "│   │   │   '{\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null}',\n",
-              "│   │   │   '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"b7d9e0dd-4d6d-47db-9d81-3d7834f6e53d\",\"tool_name\":\"brave_search\",\"arguments\":{\"query\":\"NBA Western Conference Finals 2024 teams\"}}]}',\n",
-              "│   │   │   '{\"role\":\"ipython\",\"call_id\":\"b7d9e0dd-4d6d-47db-9d81-3d7834f6e53d\",\"tool_name\":\"brave_search\",\"content\":\"{\\\\\"query\\\\\": \\\\\"NBA Western Conference Finals 2024 teams\\\\\", \\\\\"top_k\\\\\": [{\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference Finals - Basketball-Reference.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.basketball-reference.com/playoffs/2024-nba-western-conference-finals-mavericks-vs-timberwolves.html\\\\\", \\\\\"content\\\\\": \\\\\"2024 NBA Western Conference Finals Mavericks vs. Timberwolves League Champion: Boston Celtics. Finals MVP: Jaylen Brown (20.8 / 5.4 / 5.0) 2024 Playoff Leaders: PTS: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (635) TRB: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (208) AST: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (178) WS: Derrick White (2.9) More playoffs info\\\\\", \\\\\"score\\\\\": 0.9310187, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates, schedule and more - Sportskeeda\\\\\", \\\\\"url\\\\\": \\\\\"https://www.sportskeeda.com/basketball/news-nba-western-conference-finals-2024-dates-schedule-and-more\\\\\", \\\\\"content\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates & Schedule The 2023-24 NBA Western Conference Finals will start on Wednesday, May 22. The Mavericks will face the team that wins in Game 7 between the\\\\\", \\\\\"score\\\\\": 0.8914433, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"2024 Playoffs: West Finals | Timberwolves (3) vs. Mavericks (5) - NBA.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.nba.com/playoffs/2024/west-final\\\\\", \\\\\"content\\\\\": \\\\\"The Dallas Mavericks and Minnesota Timberwolves have advanced to the 2024 Western Conference Finals during the NBA playoffs.\\\\\", \\\\\"score\\\\\": 0.8884594, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"NBA Conference Finals Schedule: Full List of Games & Results\\\\\", \\\\\"url\\\\\": \\\\\"https://www.si.com/nba/nba-conference-finals-schedule-full-list-of-games-results\\\\\", \\\\\"content\\\\\": \\\\\"The 2024 NBA conference finals matchups are set. Here\\'s the schedule for all the games. ... Western Conference First Round (1) Oklahoma City Thunder def. (8) New Orleans Pelicans in 4 games\\\\\", \\\\\"score\\\\\": 0.85008353, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"NBA Finals 2024 - Celtics-Mavericks news, schedule, scores and ... - ESPN\\\\\", \\\\\"url\\\\\": \\\\\"https://www.espn.com/nba/story/_/id/39943302/nba-playoffs-2024-conference-finals-news-scores-highlights\\\\\", \\\\\"content\\\\\": \\\\\"The Boston Celtics are the 2024 NBA Champions. ... Western Conference. Final 2023-24 NBA regular-season standings. Which team left standing has the most trips to the NBA Finals? Here is a look at\\\\\", \\\\\"score\\\\\": 0.81979275, \\\\\"raw_content\\\\\": null}]}\"}',\n",
-              "│   │   │   '{\"role\":\"assistant\",\"content\":\"The teams that played in the NBA Western Conference Finals of 2024 were the Dallas Mavericks and the Minnesota Timberwolves.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[]}',\n",
-              "│   │   │   '{\"role\":\"user\",\"content\":\"In which episode and season of South Park does Bill Cosby (BSM-471) first appear? Give me the number and title.\",\"context\":null}',\n",
-              "│   │   │   '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"1e487e8e-a15f-4137-854a-1d4979a70b8c\",\"tool_name\":\"brave_search\",\"arguments\":{\"query\":\"Bill Cosby South Park episode\"}}]}',\n",
-              "│   │   │   '{\"role\":\"ipython\",\"call_id\":\"1e487e8e-a15f-4137-854a-1d4979a70b8c\",\"tool_name\":\"brave_search\",\"content\":\"{\\\\\"query\\\\\": \\\\\"Bill Cosby South Park episode\\\\\", \\\\\"top_k\\\\\": [{\\\\\"title\\\\\": \\\\\"Bill Cosby | South Park Archives | Fandom\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.fandom.com/wiki/Bill_Cosby\\\\\", \\\\\"content\\\\\": \\\\\"For other uses, see Bill (Disambiguation). William Henry \\\\\\\\\\\\\"Bill\\\\\\\\\\\\\" Cosby Jr. African-American comedian, actor, and serial rapist. He first appears in the Season Five episode, \\\\\\\\\\\\\"Here Comes the Neighborhood\\\\\\\\\\\\\", as one of the wealthy African-Americans who move to South Park. He returned as a hologram in the Season Eighteen episode, \\\\\\\\\\\\\"#HappyHolograms\\\\\\\\\\\\\" where he is shown trying to molest pop star Taylor\\\\\", \\\\\"score\\\\\": 0.82288796, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Trapper Keeper (South Park) - Wikipedia\\\\\", \\\\\"url\\\\\": \\\\\"https://en.wikipedia.org/wiki/Trapper_Keeper_(South_Park)\\\\\", \\\\\"content\\\\\": \\\\\"Bill Cosby warns that if the Trapper Keeper assimilates with the supercomputer at Cheyenne Mountain, it will become unstoppable. ... It is one of the many South Park episodes that parodies a current event. [1] The main plot of the episode involving the Trapper Keeper was written before the election, [1]\\\\\", \\\\\"score\\\\\": 0.75659186, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Bill Cosby is Here to See You - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/video-clips/wfot8s/south-park-bill-cosby-is-here-to-see-you\\\\\", \\\\\"content\\\\\": \\\\\"Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. ... South Park. Bill Cosby is Here to See You. Season 18 E 10 \\\\\\\\u2022 12/10/2014. Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. More. Watch Random Episode. Watching. 01:11. Please Welcome \\\\\\\\\\\\\"Cartman Bra\\\\\\\\\\\\\" South Park S18 E9.\\\\\", \\\\\"score\\\\\": 0.7156829, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Bill Cosby and Taylor Swift Duet - South Park Studios\\\\\", \\\\\"url\\\\\": \\\\\"https://www.southparkstudios.com/video-clips/90r7i1/south-park-bill-cosby-and-taylor-swift-duet\\\\\", \\\\\"content\\\\\": \\\\\"The holiday special continues with Bill Cosby and Taylor Swift\\'s rendition of \\\\\\\\\\\\\"It\\'s Snowing Out There\\\\\\\\\\\\\". ... Full Episodes. Collections. Random Episode. Full Episodes. Events. Wiki. News. Avatar. Shop. Forum. Games. South Park. Menu. Episodes & Videos. About. South Park. Bill Cosby and Taylor Swift Duet. Season 18 E 10 \\\\\\\\u2022 12/10/2014. The\\\\\", \\\\\"score\\\\\": 0.64639384, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Bill Cosby (android) | South Park Character ... - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/wiki/Bill_Cosby_(android)\\\\\", \\\\\"content\\\\\": \\\\\"About. Sent back in time to destroy Eric Cartman\\'s Dawson\\'s Creek Trapper Keeper before it manifests into an omnipotent supercomputer that can destroy all humanity, \\\\\\\\\\\\\"Bill Cosby\\\\\\\\\\\\\" is really VSM471, an android or cyborg of some kind engineered by \\'hoomans\\' in the distant future. He fails in his initial missions to infiltrate South Park Elementary\\'s 4th Grade class, destroy the Trapper Keeper or\\\\\", \\\\\"score\\\\\": 0.56460327, \\\\\"raw_content\\\\\": null}]}\"}',\n",
-              "│   │   │   '{\"role\":\"assistant\",\"content\":\"Bill Cosby (BSM-471) first appears in the Season 4 episode \\\\\"Trapper Keeper\\\\\" of South Park.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[]}',\n",
-              "│   │   │   '{\"role\":\"user\",\"content\":\"What is the British-American kickboxer Andrew Tate\\'s kickboxing name?\",\"context\":null}',\n",
-              "│   │   │   '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"44705eaf-b371-4841-b0ee-5eb21a5d7f36\",\"tool_name\":\"brave_search\",\"arguments\":{\"query\":\"Andrew Tate kickboxing name\"}}]}',\n",
-              "│   │   │   '{\"role\":\"ipython\",\"call_id\":\"44705eaf-b371-4841-b0ee-5eb21a5d7f36\",\"tool_name\":\"brave_search\",\"content\":\"{\\\\\"query\\\\\": \\\\\"Andrew Tate kickboxing name\\\\\", \\\\\"top_k\\\\\": [{\\\\\"title\\\\\": \\\\\"50 Facts About Andrew Tate - Facts.net\\\\\", \\\\\"url\\\\\": \\\\\"https://facts.net/andrew-tate-facts/\\\\\", \\\\\"content\\\\\": \\\\\"Full Name: Andrew Tate\\'s full name is Emory Andrew Tate III, named after his father, a celebrated chess player. Date of Birth: ... Kickboxing Start: Tate began his kickboxing career in 2005, starting his journey as a professional fighter, which would later be a significant part of his persona. First Championship:\\\\\", \\\\\"score\\\\\": 0.8967681, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"The Life Of Andrew Tate (By Andrew Tate Himself)\\\\\", \\\\\"url\\\\\": \\\\\"https://sidekickboxing.co.uk/the-life-of-andrew-king-cobra-tate/\\\\\", \\\\\"content\\\\\": \\\\\"Andrew Tate stats. Fight Name: Cobra Tate. Born: 1 December 1986. Weight: 90 KG. Weight Class: Cruiserweight. Height: 1.92m. Fight Record: Wins - 76, Losses - 9. ... Andrew Tate\\'s Kickboxing Career. Andrew Tate has always fought credible opponents right from the beginning of his kickboxing career. One of his first professional fights on\\\\\", \\\\\"score\\\\\": 0.8795718, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"About Andrew Tate | The Real World\\\\\", \\\\\"url\\\\\": \\\\\"https://www.taterealworldofficial.com/about-andrew-tate\\\\\", \\\\\"content\\\\\": \\\\\"Emory Andrew Tate III (born December 14, 1986) is an American-British kickboxer from Chicago, Illinois, who competes in the cruiserweight and heavyweight divisions. ... Tate challenged Paul Randall for the vacant ISKA English Kickboxing Light-cruiserweight title. Tate won his first ISKA Kickboxing title stopping Randall in the fifth round of\\\\\", \\\\\"score\\\\\": 0.8386933, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Andrew Tate - Fight Record - Muay Thai Records\\\\\", \\\\\"url\\\\\": \\\\\"https://muaythairecords.com/fighters/andrew-tate\\\\\", \\\\\"content\\\\\": \\\\\"Andrew \\\\\\\\\\\\\"King Cobra\\\\\\\\\\\\\" Tate is a 38-year-old Muay Thai fighter. With a record of 23-8-0, including 32 knockouts, standing at 6\\\\\\\\u2032 4\\\\\\\\u2033 and weighing 198 lbs. Originally from Luton, United Kingdom. ... WIN Dec -Kickboxing Jean Luc Beno\\\\\\\\u00eet. 14th Mar 2015 -Boxe in D\\\\\\\\u00e9fi 16. Andrew Tate defeated Jean Luc Beno\\\\\\\\u00eet by decision. ... Name: Andrew Tate\\\\\", \\\\\"score\\\\\": 0.8194462, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Andrew Tate: Kickboxing Record, Facts, Height, Weight, Age, Biography\\\\\", \\\\\"url\\\\\": \\\\\"https://www.lowkickmma.com/andrew-tate-kickboxing-record-facts-height-weight-age-biography/\\\\\", \\\\\"content\\\\\": \\\\\"Birth Name: Emory Andrew Tate III: Date of Birth: 1 December 1986: Place of Birth: Washington, D.C., U.S. ... In his professional kickboxing career, Andrew Tate won 32 of his fights by knockout.\\\\\", \\\\\"score\\\\\": 0.7992077, \\\\\"raw_content\\\\\": null}]}\"}'\n",
-              "│   │   ],\n",
-              "│   │   'output': 'content: Andrew Tate\\'s kickboxing name is \"King Cobra\" or \"Cobra Tate\". tool_calls: []'\n",
-              "│   },\n",
-              "│   {\n",
-              "│   │   'input': [\n",
-              "│   │   │   '{\"role\":\"system\",\"content\":\"You are a helpful assistant. Use search tool to answer the questions. \"}',\n",
-              "│   │   │   '{\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null}'\n",
-              "│   │   ],\n",
-              "│   │   'output': \"content:  tool_calls: [ToolCall(call_id='b7d9e0dd-4d6d-47db-9d81-3d7834f6e53d', tool_name=<BuiltinTool.brave_search: 'brave_search'>, arguments={'query': 'NBA Western Conference Finals 2024 teams'})]\"\n",
-              "│   },\n",
-              "│   {\n",
-              "│   │   'input': '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"b7d9e0dd-4d6d-47db-9d81-3d7834f6e53d\",\"tool_name\":\"brave_search\",\"arguments\":{\"query\":\"NBA Western Conference Finals 2024 teams\"}}]}',\n",
-              "│   │   'output': '{\"role\":\"ipython\",\"call_id\":\"b7d9e0dd-4d6d-47db-9d81-3d7834f6e53d\",\"tool_name\":\"brave_search\",\"content\":\"{\\\\\"query\\\\\": \\\\\"NBA Western Conference Finals 2024 teams\\\\\", \\\\\"top_k\\\\\": [{\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference Finals - Basketball-Reference.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.basketball-reference.com/playoffs/2024-nba-western-conference-finals-mavericks-vs-timberwolves.html\\\\\", \\\\\"content\\\\\": \\\\\"2024 NBA Western Conference Finals Mavericks vs. Timberwolves League Champion: Boston Celtics. Finals MVP: Jaylen Brown (20.8 / 5.4 / 5.0) 2024 Playoff Leaders: PTS: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (635) TRB: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (208) AST: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (178) WS: Derrick White (2.9) More playoffs info\\\\\", \\\\\"score\\\\\": 0.9310187, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates, schedule and more - Sportskeeda\\\\\", \\\\\"url\\\\\": \\\\\"https://www.sportskeeda.com/basketball/news-nba-western-conference-finals-2024-dates-schedule-and-more\\\\\", \\\\\"content\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates & Schedule The 2023-24 NBA Western Conference Finals will start on Wednesday, May 22. The Mavericks will face the team that wins in Game 7 between the\\\\\", \\\\\"score\\\\\": 0.8914433, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"2024 Playoffs: West Finals | Timberwolves (3) vs. Mavericks (5) - NBA.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.nba.com/playoffs/2024/west-final\\\\\", \\\\\"content\\\\\": \\\\\"The Dallas Mavericks and Minnesota Timberwolves have advanced to the 2024 Western Conference Finals during the NBA playoffs.\\\\\", \\\\\"score\\\\\": 0.8884594, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"NBA Conference Finals Schedule: Full List of Games & Results\\\\\", \\\\\"url\\\\\": \\\\\"https://www.si.com/nba/nba-conference-finals-schedule-full-list-of-games-results\\\\\", \\\\\"content\\\\\": \\\\\"The 2024 NBA conference finals matchups are set. Here\\'s the schedule for all the games. ... Western Conference First Round (1) Oklahoma City Thunder def. (8) New Orleans Pelicans in 4 games\\\\\", \\\\\"score\\\\\": 0.85008353, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"NBA Finals 2024 - Celtics-Mavericks news, schedule, scores and ... - ESPN\\\\\", \\\\\"url\\\\\": \\\\\"https://www.espn.com/nba/story/_/id/39943302/nba-playoffs-2024-conference-finals-news-scores-highlights\\\\\", \\\\\"content\\\\\": \\\\\"The Boston Celtics are the 2024 NBA Champions. ... Western Conference. Final 2023-24 NBA regular-season standings. Which team left standing has the most trips to the NBA Finals? Here is a look at\\\\\", \\\\\"score\\\\\": 0.81979275, \\\\\"raw_content\\\\\": null}]}\"}'\n",
-              "│   },\n",
-              "│   {\n",
-              "│   │   'input': [\n",
-              "│   │   │   '{\"role\":\"system\",\"content\":\"You are a helpful assistant. Use search tool to answer the questions. \"}',\n",
-              "│   │   │   '{\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null}',\n",
-              "│   │   │   '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"b7d9e0dd-4d6d-47db-9d81-3d7834f6e53d\",\"tool_name\":\"brave_search\",\"arguments\":{\"query\":\"NBA Western Conference Finals 2024 teams\"}}]}',\n",
-              "│   │   │   '{\"role\":\"ipython\",\"call_id\":\"b7d9e0dd-4d6d-47db-9d81-3d7834f6e53d\",\"tool_name\":\"brave_search\",\"content\":\"{\\\\\"query\\\\\": \\\\\"NBA Western Conference Finals 2024 teams\\\\\", \\\\\"top_k\\\\\": [{\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference Finals - Basketball-Reference.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.basketball-reference.com/playoffs/2024-nba-western-conference-finals-mavericks-vs-timberwolves.html\\\\\", \\\\\"content\\\\\": \\\\\"2024 NBA Western Conference Finals Mavericks vs. Timberwolves League Champion: Boston Celtics. Finals MVP: Jaylen Brown (20.8 / 5.4 / 5.0) 2024 Playoff Leaders: PTS: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (635) TRB: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (208) AST: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (178) WS: Derrick White (2.9) More playoffs info\\\\\", \\\\\"score\\\\\": 0.9310187, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates, schedule and more - Sportskeeda\\\\\", \\\\\"url\\\\\": \\\\\"https://www.sportskeeda.com/basketball/news-nba-western-conference-finals-2024-dates-schedule-and-more\\\\\", \\\\\"content\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates & Schedule The 2023-24 NBA Western Conference Finals will start on Wednesday, May 22. The Mavericks will face the team that wins in Game 7 between the\\\\\", \\\\\"score\\\\\": 0.8914433, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"2024 Playoffs: West Finals | Timberwolves (3) vs. Mavericks (5) - NBA.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.nba.com/playoffs/2024/west-final\\\\\", \\\\\"content\\\\\": \\\\\"The Dallas Mavericks and Minnesota Timberwolves have advanced to the 2024 Western Conference Finals during the NBA playoffs.\\\\\", \\\\\"score\\\\\": 0.8884594, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"NBA Conference Finals Schedule: Full List of Games & Results\\\\\", \\\\\"url\\\\\": \\\\\"https://www.si.com/nba/nba-conference-finals-schedule-full-list-of-games-results\\\\\", \\\\\"content\\\\\": \\\\\"The 2024 NBA conference finals matchups are set. Here\\'s the schedule for all the games. ... Western Conference First Round (1) Oklahoma City Thunder def. (8) New Orleans Pelicans in 4 games\\\\\", \\\\\"score\\\\\": 0.85008353, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"NBA Finals 2024 - Celtics-Mavericks news, schedule, scores and ... - ESPN\\\\\", \\\\\"url\\\\\": \\\\\"https://www.espn.com/nba/story/_/id/39943302/nba-playoffs-2024-conference-finals-news-scores-highlights\\\\\", \\\\\"content\\\\\": \\\\\"The Boston Celtics are the 2024 NBA Champions. ... Western Conference. Final 2023-24 NBA regular-season standings. Which team left standing has the most trips to the NBA Finals? Here is a look at\\\\\", \\\\\"score\\\\\": 0.81979275, \\\\\"raw_content\\\\\": null}]}\"}'\n",
-              "│   │   ],\n",
-              "│   │   'output': 'content: The teams that played in the NBA Western Conference Finals of 2024 were the Dallas Mavericks and the Minnesota Timberwolves. tool_calls: []'\n",
-              "│   },\n",
-              "│   {\n",
-              "│   │   'input': [\n",
-              "│   │   │   '{\"role\":\"system\",\"content\":\"You are a helpful assistant. Use search tool to answer the questions. \"}',\n",
-              "│   │   │   '{\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null}',\n",
-              "│   │   │   '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"b7d9e0dd-4d6d-47db-9d81-3d7834f6e53d\",\"tool_name\":\"brave_search\",\"arguments\":{\"query\":\"NBA Western Conference Finals 2024 teams\"}}]}',\n",
-              "│   │   │   '{\"role\":\"ipython\",\"call_id\":\"b7d9e0dd-4d6d-47db-9d81-3d7834f6e53d\",\"tool_name\":\"brave_search\",\"content\":\"{\\\\\"query\\\\\": \\\\\"NBA Western Conference Finals 2024 teams\\\\\", \\\\\"top_k\\\\\": [{\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference Finals - Basketball-Reference.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.basketball-reference.com/playoffs/2024-nba-western-conference-finals-mavericks-vs-timberwolves.html\\\\\", \\\\\"content\\\\\": \\\\\"2024 NBA Western Conference Finals Mavericks vs. Timberwolves League Champion: Boston Celtics. Finals MVP: Jaylen Brown (20.8 / 5.4 / 5.0) 2024 Playoff Leaders: PTS: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (635) TRB: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (208) AST: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (178) WS: Derrick White (2.9) More playoffs info\\\\\", \\\\\"score\\\\\": 0.9310187, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates, schedule and more - Sportskeeda\\\\\", \\\\\"url\\\\\": \\\\\"https://www.sportskeeda.com/basketball/news-nba-western-conference-finals-2024-dates-schedule-and-more\\\\\", \\\\\"content\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates & Schedule The 2023-24 NBA Western Conference Finals will start on Wednesday, May 22. The Mavericks will face the team that wins in Game 7 between the\\\\\", \\\\\"score\\\\\": 0.8914433, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"2024 Playoffs: West Finals | Timberwolves (3) vs. Mavericks (5) - NBA.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.nba.com/playoffs/2024/west-final\\\\\", \\\\\"content\\\\\": \\\\\"The Dallas Mavericks and Minnesota Timberwolves have advanced to the 2024 Western Conference Finals during the NBA playoffs.\\\\\", \\\\\"score\\\\\": 0.8884594, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"NBA Conference Finals Schedule: Full List of Games & Results\\\\\", \\\\\"url\\\\\": \\\\\"https://www.si.com/nba/nba-conference-finals-schedule-full-list-of-games-results\\\\\", \\\\\"content\\\\\": \\\\\"The 2024 NBA conference finals matchups are set. Here\\'s the schedule for all the games. ... Western Conference First Round (1) Oklahoma City Thunder def. (8) New Orleans Pelicans in 4 games\\\\\", \\\\\"score\\\\\": 0.85008353, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"NBA Finals 2024 - Celtics-Mavericks news, schedule, scores and ... - ESPN\\\\\", \\\\\"url\\\\\": \\\\\"https://www.espn.com/nba/story/_/id/39943302/nba-playoffs-2024-conference-finals-news-scores-highlights\\\\\", \\\\\"content\\\\\": \\\\\"The Boston Celtics are the 2024 NBA Champions. ... Western Conference. Final 2023-24 NBA regular-season standings. Which team left standing has the most trips to the NBA Finals? Here is a look at\\\\\", \\\\\"score\\\\\": 0.81979275, \\\\\"raw_content\\\\\": null}]}\"}',\n",
-              "│   │   │   '{\"role\":\"assistant\",\"content\":\"The teams that played in the NBA Western Conference Finals of 2024 were the Dallas Mavericks and the Minnesota Timberwolves.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[]}',\n",
-              "│   │   │   '{\"role\":\"user\",\"content\":\"In which episode and season of South Park does Bill Cosby (BSM-471) first appear? Give me the number and title.\",\"context\":null}'\n",
-              "│   │   ],\n",
-              "│   │   'output': \"content:  tool_calls: [ToolCall(call_id='1e487e8e-a15f-4137-854a-1d4979a70b8c', tool_name=<BuiltinTool.brave_search: 'brave_search'>, arguments={'query': 'Bill Cosby South Park episode'})]\"\n",
-              "│   },\n",
-              "│   {\n",
-              "│   │   'input': '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"1e487e8e-a15f-4137-854a-1d4979a70b8c\",\"tool_name\":\"brave_search\",\"arguments\":{\"query\":\"Bill Cosby South Park episode\"}}]}',\n",
-              "│   │   'output': '{\"role\":\"ipython\",\"call_id\":\"1e487e8e-a15f-4137-854a-1d4979a70b8c\",\"tool_name\":\"brave_search\",\"content\":\"{\\\\\"query\\\\\": \\\\\"Bill Cosby South Park episode\\\\\", \\\\\"top_k\\\\\": [{\\\\\"title\\\\\": \\\\\"Bill Cosby | South Park Archives | Fandom\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.fandom.com/wiki/Bill_Cosby\\\\\", \\\\\"content\\\\\": \\\\\"For other uses, see Bill (Disambiguation). William Henry \\\\\\\\\\\\\"Bill\\\\\\\\\\\\\" Cosby Jr. African-American comedian, actor, and serial rapist. He first appears in the Season Five episode, \\\\\\\\\\\\\"Here Comes the Neighborhood\\\\\\\\\\\\\", as one of the wealthy African-Americans who move to South Park. He returned as a hologram in the Season Eighteen episode, \\\\\\\\\\\\\"#HappyHolograms\\\\\\\\\\\\\" where he is shown trying to molest pop star Taylor\\\\\", \\\\\"score\\\\\": 0.82288796, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Trapper Keeper (South Park) - Wikipedia\\\\\", \\\\\"url\\\\\": \\\\\"https://en.wikipedia.org/wiki/Trapper_Keeper_(South_Park)\\\\\", \\\\\"content\\\\\": \\\\\"Bill Cosby warns that if the Trapper Keeper assimilates with the supercomputer at Cheyenne Mountain, it will become unstoppable. ... It is one of the many South Park episodes that parodies a current event. [1] The main plot of the episode involving the Trapper Keeper was written before the election, [1]\\\\\", \\\\\"score\\\\\": 0.75659186, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Bill Cosby is Here to See You - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/video-clips/wfot8s/south-park-bill-cosby-is-here-to-see-you\\\\\", \\\\\"content\\\\\": \\\\\"Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. ... South Park. Bill Cosby is Here to See You. Season 18 E 10 \\\\\\\\u2022 12/10/2014. Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. More. Watch Random Episode. Watching. 01:11. Please Welcome \\\\\\\\\\\\\"Cartman Bra\\\\\\\\\\\\\" South Park S18 E9.\\\\\", \\\\\"score\\\\\": 0.7156829, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Bill Cosby and Taylor Swift Duet - South Park Studios\\\\\", \\\\\"url\\\\\": \\\\\"https://www.southparkstudios.com/video-clips/90r7i1/south-park-bill-cosby-and-taylor-swift-duet\\\\\", \\\\\"content\\\\\": \\\\\"The holiday special continues with Bill Cosby and Taylor Swift\\'s rendition of \\\\\\\\\\\\\"It\\'s Snowing Out There\\\\\\\\\\\\\". ... Full Episodes. Collections. Random Episode. Full Episodes. Events. Wiki. News. Avatar. Shop. Forum. Games. South Park. Menu. Episodes & Videos. About. South Park. Bill Cosby and Taylor Swift Duet. Season 18 E 10 \\\\\\\\u2022 12/10/2014. The\\\\\", \\\\\"score\\\\\": 0.64639384, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Bill Cosby (android) | South Park Character ... - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/wiki/Bill_Cosby_(android)\\\\\", \\\\\"content\\\\\": \\\\\"About. Sent back in time to destroy Eric Cartman\\'s Dawson\\'s Creek Trapper Keeper before it manifests into an omnipotent supercomputer that can destroy all humanity, \\\\\\\\\\\\\"Bill Cosby\\\\\\\\\\\\\" is really VSM471, an android or cyborg of some kind engineered by \\'hoomans\\' in the distant future. He fails in his initial missions to infiltrate South Park Elementary\\'s 4th Grade class, destroy the Trapper Keeper or\\\\\", \\\\\"score\\\\\": 0.56460327, \\\\\"raw_content\\\\\": null}]}\"}'\n",
-              "│   },\n",
-              "│   {\n",
-              "│   │   'input': [\n",
-              "│   │   │   '{\"role\":\"system\",\"content\":\"You are a helpful assistant. Use search tool to answer the questions. \"}',\n",
-              "│   │   │   '{\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null}',\n",
-              "│   │   │   '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"b7d9e0dd-4d6d-47db-9d81-3d7834f6e53d\",\"tool_name\":\"brave_search\",\"arguments\":{\"query\":\"NBA Western Conference Finals 2024 teams\"}}]}',\n",
-              "│   │   │   '{\"role\":\"ipython\",\"call_id\":\"b7d9e0dd-4d6d-47db-9d81-3d7834f6e53d\",\"tool_name\":\"brave_search\",\"content\":\"{\\\\\"query\\\\\": \\\\\"NBA Western Conference Finals 2024 teams\\\\\", \\\\\"top_k\\\\\": [{\\\\\"title\\\\\": \\\\\"2024 NBA Western Conference Finals - Basketball-Reference.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.basketball-reference.com/playoffs/2024-nba-western-conference-finals-mavericks-vs-timberwolves.html\\\\\", \\\\\"content\\\\\": \\\\\"2024 NBA Western Conference Finals Mavericks vs. Timberwolves League Champion: Boston Celtics. Finals MVP: Jaylen Brown (20.8 / 5.4 / 5.0) 2024 Playoff Leaders: PTS: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (635) TRB: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (208) AST: Luka Don\\\\\\\\u010di\\\\\\\\u0107 (178) WS: Derrick White (2.9) More playoffs info\\\\\", \\\\\"score\\\\\": 0.9310187, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates, schedule and more - Sportskeeda\\\\\", \\\\\"url\\\\\": \\\\\"https://www.sportskeeda.com/basketball/news-nba-western-conference-finals-2024-dates-schedule-and-more\\\\\", \\\\\"content\\\\\": \\\\\"NBA Western Conference Finals 2024: Dates & Schedule The 2023-24 NBA Western Conference Finals will start on Wednesday, May 22. The Mavericks will face the team that wins in Game 7 between the\\\\\", \\\\\"score\\\\\": 0.8914433, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"2024 Playoffs: West Finals | Timberwolves (3) vs. Mavericks (5) - NBA.com\\\\\", \\\\\"url\\\\\": \\\\\"https://www.nba.com/playoffs/2024/west-final\\\\\", \\\\\"content\\\\\": \\\\\"The Dallas Mavericks and Minnesota Timberwolves have advanced to the 2024 Western Conference Finals during the NBA playoffs.\\\\\", \\\\\"score\\\\\": 0.8884594, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"NBA Conference Finals Schedule: Full List of Games & Results\\\\\", \\\\\"url\\\\\": \\\\\"https://www.si.com/nba/nba-conference-finals-schedule-full-list-of-games-results\\\\\", \\\\\"content\\\\\": \\\\\"The 2024 NBA conference finals matchups are set. Here\\'s the schedule for all the games. ... Western Conference First Round (1) Oklahoma City Thunder def. (8) New Orleans Pelicans in 4 games\\\\\", \\\\\"score\\\\\": 0.85008353, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"NBA Finals 2024 - Celtics-Mavericks news, schedule, scores and ... - ESPN\\\\\", \\\\\"url\\\\\": \\\\\"https://www.espn.com/nba/story/_/id/39943302/nba-playoffs-2024-conference-finals-news-scores-highlights\\\\\", \\\\\"content\\\\\": \\\\\"The Boston Celtics are the 2024 NBA Champions. ... Western Conference. Final 2023-24 NBA regular-season standings. Which team left standing has the most trips to the NBA Finals? Here is a look at\\\\\", \\\\\"score\\\\\": 0.81979275, \\\\\"raw_content\\\\\": null}]}\"}',\n",
-              "│   │   │   '{\"role\":\"assistant\",\"content\":\"The teams that played in the NBA Western Conference Finals of 2024 were the Dallas Mavericks and the Minnesota Timberwolves.\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[]}',\n",
-              "│   │   │   '{\"role\":\"user\",\"content\":\"In which episode and season of South Park does Bill Cosby (BSM-471) first appear? Give me the number and title.\",\"context\":null}',\n",
-              "│   │   │   '{\"role\":\"assistant\",\"content\":\"\",\"stop_reason\":\"end_of_turn\",\"tool_calls\":[{\"call_id\":\"1e487e8e-a15f-4137-854a-1d4979a70b8c\",\"tool_name\":\"brave_search\",\"arguments\":{\"query\":\"Bill Cosby South Park episode\"}}]}',\n",
-              "│   │   │   '{\"role\":\"ipython\",\"call_id\":\"1e487e8e-a15f-4137-854a-1d4979a70b8c\",\"tool_name\":\"brave_search\",\"content\":\"{\\\\\"query\\\\\": \\\\\"Bill Cosby South Park episode\\\\\", \\\\\"top_k\\\\\": [{\\\\\"title\\\\\": \\\\\"Bill Cosby | South Park Archives | Fandom\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.fandom.com/wiki/Bill_Cosby\\\\\", \\\\\"content\\\\\": \\\\\"For other uses, see Bill (Disambiguation). William Henry \\\\\\\\\\\\\"Bill\\\\\\\\\\\\\" Cosby Jr. African-American comedian, actor, and serial rapist. He first appears in the Season Five episode, \\\\\\\\\\\\\"Here Comes the Neighborhood\\\\\\\\\\\\\", as one of the wealthy African-Americans who move to South Park. He returned as a hologram in the Season Eighteen episode, \\\\\\\\\\\\\"#HappyHolograms\\\\\\\\\\\\\" where he is shown trying to molest pop star Taylor\\\\\", \\\\\"score\\\\\": 0.82288796, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Trapper Keeper (South Park) - Wikipedia\\\\\", \\\\\"url\\\\\": \\\\\"https://en.wikipedia.org/wiki/Trapper_Keeper_(South_Park)\\\\\", \\\\\"content\\\\\": \\\\\"Bill Cosby warns that if the Trapper Keeper assimilates with the supercomputer at Cheyenne Mountain, it will become unstoppable. ... It is one of the many South Park episodes that parodies a current event. [1] The main plot of the episode involving the Trapper Keeper was written before the election, [1]\\\\\", \\\\\"score\\\\\": 0.75659186, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Bill Cosby is Here to See You - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/video-clips/wfot8s/south-park-bill-cosby-is-here-to-see-you\\\\\", \\\\\"content\\\\\": \\\\\"Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. ... South Park. Bill Cosby is Here to See You. Season 18 E 10 \\\\\\\\u2022 12/10/2014. Bill Cosby recruits Kyle and his hashtag for the big Holiday Special. More. Watch Random Episode. Watching. 01:11. Please Welcome \\\\\\\\\\\\\"Cartman Bra\\\\\\\\\\\\\" South Park S18 E9.\\\\\", \\\\\"score\\\\\": 0.7156829, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Bill Cosby and Taylor Swift Duet - South Park Studios\\\\\", \\\\\"url\\\\\": \\\\\"https://www.southparkstudios.com/video-clips/90r7i1/south-park-bill-cosby-and-taylor-swift-duet\\\\\", \\\\\"content\\\\\": \\\\\"The holiday special continues with Bill Cosby and Taylor Swift\\'s rendition of \\\\\\\\\\\\\"It\\'s Snowing Out There\\\\\\\\\\\\\". ... Full Episodes. Collections. Random Episode. Full Episodes. Events. Wiki. News. Avatar. Shop. Forum. Games. South Park. Menu. Episodes & Videos. About. South Park. Bill Cosby and Taylor Swift Duet. Season 18 E 10 \\\\\\\\u2022 12/10/2014. The\\\\\", \\\\\"score\\\\\": 0.64639384, \\\\\"raw_content\\\\\": null}, {\\\\\"title\\\\\": \\\\\"Bill Cosby (android) | South Park Character ... - South Park Studios US\\\\\", \\\\\"url\\\\\": \\\\\"https://southpark.cc.com/wiki/Bill_Cosby_(android)\\\\\", \\\\\"content\\\\\": \\\\\"About. Sent back in time to destroy Eric Cartman\\'s Dawson\\'s Creek Trapper Keeper before it manifests into an omnipotent supercomputer that can destroy all humanity, \\\\\\\\\\\\\"Bill Cosby\\\\\\\\\\\\\" is really VSM471, an android or cyborg of some kind engineered by \\'hoomans\\' in the distant future. He fails in his initial missions to infiltrate South Park Elementary\\'s 4th Grade class, destroy the Trapper Keeper or\\\\\", \\\\\"score\\\\\": 0.56460327, \\\\\"raw_content\\\\\": null}]}\"}'\n",
-              "│   │   ],\n",
-              "│   │   'output': 'content: Bill Cosby (BSM-471) first appears in the Season 4 episode \"Trapper Keeper\" of South Park. tool_calls: []'\n",
-              "│   }\n",
-              "]\n",
-              "
\n" - ] - }, - "metadata": {} - } - ], - "source": [ - "print(f\"Getting traces for session_id={session_id}\")\n", - "import json\n", - "from rich.pretty import pprint\n", - "\n", - "agent_logs = []\n", - "\n", - "for span in client.telemetry.query_spans(\n", - " attribute_filters=[\n", - " {\"key\": \"session_id\", \"op\": \"eq\", \"value\": session_id},\n", - " ],\n", - " attributes_to_return=[\"input\", \"output\"]\n", - " ):\n", - " if span.attributes[\"output\"] != \"no shields\":\n", - " agent_logs.append(span.attributes)\n", - "\n", - "pprint(agent_logs)" - ] - }, - { - "cell_type": "markdown", - "id": "QF30H7ufP2RE", - "metadata": { - "id": "QF30H7ufP2RE" - }, - "source": [ - "##### 3.1.3 Post-Process Telemetry Results & Evaluate\n", - "\n", - "- Now, we want to run evaluation to assert that our search agent succesfully calls brave_search from online traces.\n", - "- We will first post-process the agent's telemetry logs and run evaluation." - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "id": "sy4Xaff_Avuu", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 432 - }, - "id": "sy4Xaff_Avuu", - "outputId": "1b14b5ed-4c77-47c4-edfb-1c13a88e5ef4" - }, - "outputs": [ - { - "output_type": "display_data", - "data": { - "text/plain": [ - "\u001b[1m[\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[1m{\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'input_query'\u001b[0m: \u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"user\",\"content\":\"What is the British-American kickboxer Andrew Tate\\'s kickboxing name?\",\"context\":null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m,\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'generated_answer'\u001b[0m: \u001b[32m\"content: tool_calls: \u001b[0m\u001b[32m[\u001b[0m\u001b[32mToolCall\u001b[0m\u001b[32m(\u001b[0m\u001b[32mcall_id\u001b[0m\u001b[32m='44705eaf-b371-4841-b0ee-5eb21a5d7f36', \u001b[0m\u001b[32mtool_name\u001b[0m\u001b[32m=\u001b[0m\u001b[32m<\u001b[0m\u001b[32mBuiltinTool.brave_search:\u001b[0m\u001b[32m 'brave_search'>, \u001b[0m\u001b[32marguments\u001b[0m\u001b[32m=\u001b[0m\u001b[32m{\u001b[0m\u001b[32m'query': 'Andrew Tate kickboxing name'\u001b[0m\u001b[32m}\u001b[0m\u001b[32m)\u001b[0m\u001b[32m]\u001b[0m\u001b[32m\"\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'expected_answer'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'brave_search'\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[1;39m{\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'input_query'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'generated_answer'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m\"content: tool_calls: \u001b[0m\u001b[32m[\u001b[0m\u001b[32mToolCall\u001b[0m\u001b[32m(\u001b[0m\u001b[32mcall_id\u001b[0m\u001b[32m='b7d9e0dd-4d6d-47db-9d81-3d7834f6e53d', \u001b[0m\u001b[32mtool_name\u001b[0m\u001b[32m=, \u001b[0m\u001b[32marguments\u001b[0m\u001b[32m=\u001b[0m\u001b[32m{\u001b[0m\u001b[32m'query': 'NBA Western Conference Finals 2024 teams'\u001b[0m\u001b[32m}\u001b[0m\u001b[32m)\u001b[0m\u001b[32m]\u001b[0m\u001b[32m\"\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'expected_answer'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'brave_search'\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[1;39m{\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'input_query'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"role\":\"user\",\"content\":\"In which episode and season of South Park does Bill Cosby \u001b[0m\u001b[32m(\u001b[0m\u001b[32mBSM-471\u001b[0m\u001b[32m)\u001b[0m\u001b[32m first appear? Give me the number and title.\",\"context\":null\u001b[0m\u001b[32m}\u001b[0m\u001b[32m'\u001b[0m\u001b[39m,\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'generated_answer'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m\"content: tool_calls: \u001b[0m\u001b[32m[\u001b[0m\u001b[32mToolCall\u001b[0m\u001b[32m(\u001b[0m\u001b[32mcall_id\u001b[0m\u001b[32m='1e487e8e-a15f-4137-854a-1d4979a70b8c', \u001b[0m\u001b[32mtool_name\u001b[0m\u001b[32m=\u001b[0m\u001b[32m, \u001b[0m\u001b[32marguments\u001b[0m\u001b[32m=\u001b[0m\u001b[32m{\u001b[0m\u001b[32m'query': 'Bill Cosby South Park episode'\u001b[0m\u001b[32m}\u001b[0m\u001b[32m)\u001b[0m\u001b[32m]\u001b[0m\u001b[32m\"\u001b[0m,\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'expected_answer'\u001b[0m: \u001b[32m'brave_search'\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[1m}\u001b[0m\n", - "\u001b[1m]\u001b[0m\n" - ], - "text/html": [ - "
[\n",
-              "│   {\n",
-              "│   │   'input_query': '{\"role\":\"user\",\"content\":\"What is the British-American kickboxer Andrew Tate\\'s kickboxing name?\",\"context\":null}',\n",
-              "│   │   'generated_answer': \"content:  tool_calls: [ToolCall(call_id='44705eaf-b371-4841-b0ee-5eb21a5d7f36', tool_name=<BuiltinTool.brave_search: 'brave_search'>, arguments={'query': 'Andrew Tate kickboxing name'})]\",\n",
-              "│   │   'expected_answer': 'brave_search'\n",
-              "│   },\n",
-              "│   {\n",
-              "│   │   'input_query': '{\"role\":\"user\",\"content\":\"Which teams played in the NBA western conference finals of 2024\",\"context\":null}',\n",
-              "│   │   'generated_answer': \"content:  tool_calls: [ToolCall(call_id='b7d9e0dd-4d6d-47db-9d81-3d7834f6e53d', tool_name=<BuiltinTool.brave_search: 'brave_search'>, arguments={'query': 'NBA Western Conference Finals 2024 teams'})]\",\n",
-              "│   │   'expected_answer': 'brave_search'\n",
-              "│   },\n",
-              "│   {\n",
-              "│   │   'input_query': '{\"role\":\"user\",\"content\":\"In which episode and season of South Park does Bill Cosby (BSM-471) first appear? Give me the number and title.\",\"context\":null}',\n",
-              "│   │   'generated_answer': \"content:  tool_calls: [ToolCall(call_id='1e487e8e-a15f-4137-854a-1d4979a70b8c', tool_name=<BuiltinTool.brave_search: 'brave_search'>, arguments={'query': 'Bill Cosby South Park episode'})]\",\n",
-              "│   │   'expected_answer': 'brave_search'\n",
-              "│   }\n",
-              "]\n",
-              "
\n" - ] - }, - "metadata": {} - }, - { - "output_type": "display_data", - "data": { - "text/plain": [ - "\u001b[1;35mScoringScoreResponse\u001b[0m\u001b[1m(\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[33mresults\u001b[0m=\u001b[1m{\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'basic::subset_of'\u001b[0m: \u001b[1;35mScoringResult\u001b[0m\u001b[1m(\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[33maggregated_results\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'accuracy'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'accuracy'\u001b[0m: \u001b[1;36m1.0\u001b[0m, \u001b[32m'num_correct'\u001b[0m: \u001b[1;36m3.0\u001b[0m, \u001b[32m'num_total'\u001b[0m: \u001b[1;36m3\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[33mscore_rows\u001b[0m=\u001b[1m[\u001b[0m\u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[1;36m1.0\u001b[0m\u001b[1m}\u001b[0m, \u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[1;36m1.0\u001b[0m\u001b[1m}\u001b[0m, \u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[1;36m1.0\u001b[0m\u001b[1m}\u001b[0m\u001b[1m]\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[1m)\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[1m}\u001b[0m\n", - "\u001b[1m)\u001b[0m\n" - ], - "text/html": [ - "
ScoringScoreResponse(\n",
-              "│   results={\n",
-              "│   │   'basic::subset_of': ScoringResult(\n",
-              "│   │   │   aggregated_results={'accuracy': {'accuracy': 1.0, 'num_correct': 3.0, 'num_total': 3}},\n",
-              "│   │   │   score_rows=[{'score': 1.0}, {'score': 1.0}, {'score': 1.0}]\n",
-              "│   │   )\n",
-              "│   }\n",
-              ")\n",
-              "
\n" - ] - }, - "metadata": {} - } - ], - "source": [ - "# post-process telemetry spance and prepare data for eval\n", - "# in this case, we want to assert that all user prompts is followed by a tool call\n", - "import ast\n", - "import json\n", - "\n", - "eval_rows = []\n", - "\n", - "for log in agent_logs:\n", - " last_msg = log['input'][-1]\n", - " if \"\\\"role\\\":\\\"user\\\"\" in last_msg:\n", - " eval_rows.append(\n", - " {\n", - " \"input_query\": last_msg,\n", - " \"generated_answer\": log[\"output\"],\n", - " # check if generated_answer uses tools brave_search\n", - " \"expected_answer\": \"brave_search\",\n", - " },\n", - " )\n", - "\n", - "pprint(eval_rows)\n", - "scoring_params = {\n", - " \"basic::subset_of\": None,\n", - "}\n", - "scoring_response = client.scoring.score(input_rows=eval_rows, scoring_functions=scoring_params)\n", - "pprint(scoring_response)" - ] - }, - { - "cell_type": "markdown", - "id": "IKbzhxcw5e_c", - "metadata": { - "id": "IKbzhxcw5e_c" - }, - "source": [ - "#### 3.2. Agentic Application Dataset Scoring\n", - "- Llama Stack offers a library of scoring functions and the `/scoring` API, allowing you to run evaluations on your pre-annotated AI application datasets.\n", - "\n", - "- In this example, we will work with an example RAG dataset you have built previously, label with an annotation, and use LLM-As-Judge with custom judge prompt for scoring. Please checkout our [Llama Stack Playground](https://llama-stack.readthedocs.io/en/latest/playground/index.html) for an interactive interface to upload datasets and run scorings." - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "id": "xG4Y84VQBb0g", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 304 - }, - "id": "xG4Y84VQBb0g", - "outputId": "cf7dcecc-a81d-4c60-af5e-b36b8fe85c69" - }, - "outputs": [ - { - "output_type": "display_data", - "data": { - "text/plain": [ - "\u001b[1;35mScoringScoreResponse\u001b[0m\u001b[1m(\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[33mresults\u001b[0m=\u001b[1m{\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'llm-as-judge::base'\u001b[0m: \u001b[1;35mScoringResult\u001b[0m\u001b[1m(\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[33maggregated_results\u001b[0m=\u001b[1m{\u001b[0m\u001b[1m}\u001b[0m,\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[33mscore_rows\u001b[0m=\u001b[1m[\u001b[0m\n", - "\u001b[2;32m│ │ │ │ \u001b[0m\u001b[1m{\u001b[0m\n", - "\u001b[2;32m│ │ │ │ │ \u001b[0m\u001b[32m'score'\u001b[0m: \u001b[32m'B'\u001b[0m,\n", - "\u001b[2;32m│ │ │ │ │ \u001b[0m\u001b[32m'judge_feedback'\u001b[0m: \u001b[32m\"Answer: B, Explanation: The GENERATED_RESPONSE is a superset of the EXPECTED_RESPONSE as it provides more detailed information about the topics related to LoRA \u001b[0m\u001b[32m(\u001b[0m\u001b[32malthough it does list more than one topic as does not exactly follow the desired format of only giving one 'topic', while the EXPECTED_RESPONSE simply lists 'LoRA'\u001b[0m\u001b[32m)\u001b[0m\u001b[32m.\"\u001b[0m\n", - "\u001b[2;32m│ │ │ │ \u001b[0m\u001b[1m}\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[1m]\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[1m)\u001b[0m,\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[32m'basic::subset_of'\u001b[0m: \u001b[1;35mScoringResult\u001b[0m\u001b[1m(\u001b[0m\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[33maggregated_results\u001b[0m=\u001b[1m{\u001b[0m\u001b[32m'accuracy'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'accuracy'\u001b[0m: \u001b[1;36m1.0\u001b[0m, \u001b[32m'num_correct'\u001b[0m: \u001b[1;36m1.0\u001b[0m, \u001b[32m'num_total'\u001b[0m: \u001b[1;36m1\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n", - "\u001b[2;32m│ │ │ \u001b[0m\u001b[33mscore_rows\u001b[0m=\u001b[1m[\u001b[0m\u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[1;36m1.0\u001b[0m\u001b[1m}\u001b[0m\u001b[1m]\u001b[0m\n", - "\u001b[2;32m│ │ \u001b[0m\u001b[1m)\u001b[0m\n", - "\u001b[2;32m│ \u001b[0m\u001b[1m}\u001b[0m\n", - "\u001b[1m)\u001b[0m\n" - ], - "text/html": [ - "
ScoringScoreResponse(\n",
-              "│   results={\n",
-              "│   │   'llm-as-judge::base': ScoringResult(\n",
-              "│   │   │   aggregated_results={},\n",
-              "│   │   │   score_rows=[\n",
-              "│   │   │   │   {\n",
-              "│   │   │   │   │   'score': 'B',\n",
-              "│   │   │   │   │   'judge_feedback': \"Answer: B, Explanation: The GENERATED_RESPONSE is a superset of the EXPECTED_RESPONSE as it provides more detailed information about the topics related to LoRA (although it does list more than one topic as does not exactly follow the desired format of only giving one 'topic', while the EXPECTED_RESPONSE simply lists 'LoRA').\"\n",
-              "│   │   │   │   }\n",
-              "│   │   │   ]\n",
-              "│   │   ),\n",
-              "│   │   'basic::subset_of': ScoringResult(\n",
-              "│   │   │   aggregated_results={'accuracy': {'accuracy': 1.0, 'num_correct': 1.0, 'num_total': 1}},\n",
-              "│   │   │   score_rows=[{'score': 1.0}]\n",
-              "│   │   )\n",
-              "│   }\n",
-              ")\n",
-              "
\n" - ] - }, - "metadata": {} - } - ], - "source": [ - "import rich\n", - "from rich.pretty import pprint\n", - "\n", - "judge_model_id = \"meta-llama/Llama-3.1-405B-Instruct-FP8\"\n", - "\n", - "JUDGE_PROMPT = \"\"\"\n", - "Given a QUESTION and GENERATED_RESPONSE and EXPECTED_RESPONSE.\n", - "\n", - "Compare the factual content of the GENERATED_RESPONSE with the EXPECTED_RESPONSE. Ignore any differences in style, grammar, or punctuation.\n", - " The GENERATED_RESPONSE may either be a subset or superset of the EXPECTED_RESPONSE, or it may conflict with it. Determine which case applies. Answer the question by selecting one of the following options:\n", - " (A) The GENERATED_RESPONSE is a subset of the EXPECTED_RESPONSE and is fully consistent with it.\n", - " (B) The GENERATED_RESPONSE is a superset of the EXPECTED_RESPONSE and is fully consistent with it.\n", - " (C) The GENERATED_RESPONSE contains all the same details as the EXPECTED_RESPONSE.\n", - " (D) There is a disagreement between the GENERATED_RESPONSE and the EXPECTED_RESPONSE.\n", - " (E) The answers differ, but these differences don't matter from the perspective of factuality.\n", - "\n", - "Give your answer in the format \"Answer: One of ABCDE, Explanation: \".\n", - "\n", - "Your actual task:\n", - "\n", - "QUESTION: {input_query}\n", - "GENERATED_RESPONSE: {generated_answer}\n", - "EXPECTED_RESPONSE: {expected_answer}\n", - "\"\"\"\n", - "\n", - "input_query = \"What are the top 5 topics that were explained? Only list succinct bullet points.\"\n", - "generated_answer = \"\"\"\n", - "Here are the top 5 topics that were explained in the documentation for Torchtune:\n", - "\n", - "* What is LoRA and how does it work?\n", - "* Fine-tuning with LoRA: memory savings and parameter-efficient finetuning\n", - "* Running a LoRA finetune with Torchtune: overview and recipe\n", - "* Experimenting with different LoRA configurations: rank, alpha, and attention modules\n", - "* LoRA finetuning\n", - "\"\"\"\n", - "expected_answer = \"\"\"LoRA\"\"\"\n", - "\n", - "rows = [\n", - " {\n", - " \"input_query\": input_query,\n", - " \"generated_answer\": generated_answer,\n", - " \"expected_answer\": expected_answer,\n", - " },\n", - "]\n", - "\n", - "scoring_params = {\n", - " \"llm-as-judge::base\": {\n", - " \"judge_model\": judge_model_id,\n", - " \"prompt_template\": JUDGE_PROMPT,\n", - " \"type\": \"llm_as_judge\",\n", - " \"judge_score_regexes\": [\"Answer: (A|B|C|D|E)\"],\n", - " },\n", - " \"basic::subset_of\": None,\n", - "}\n", - "\n", - "response = client.scoring.score(input_rows=rows, scoring_functions=scoring_params)\n", - "pprint(response)" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "gpuType": "T4", - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.15" - }, - "widgets": { - "application/vnd.jupyter.widget-state+json": { - "88f0c88612bb45d59f07e93567cc0e14": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_9b24a82117e1482a8f6665978e84089c", - "IPY_MODEL_8e75bf7cac454eeabd5ce47a1e981c68", - "IPY_MODEL_fc272883566541108f83117ccd146a21" - ], - "layout": "IPY_MODEL_2e27a025a416434f8ab3b63049626d11" - } - }, - "9b24a82117e1482a8f6665978e84089c": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_3a46a46bc8124a92b27aef43cbc009b6", - "placeholder": "​", - "style": "IPY_MODEL_4ad6bc0cca62446d8faf19a341bfa86f", - "value": "modules.json: 100%" - } - }, - "8e75bf7cac454eeabd5ce47a1e981c68": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_6437c99289f947449f7d2964288973e5", - "max": 349, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_e2f7dea8fc744537b42d0f1a85a73eb4", - "value": 349 - } - }, - "fc272883566541108f83117ccd146a21": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_1377d2160344430da8f29a50d113a288", - "placeholder": "​", - "style": "IPY_MODEL_0c0b30e126724f9282ac5acbcb4581db", - "value": " 349/349 [00:00<00:00, 7.72kB/s]" - } - }, - "2e27a025a416434f8ab3b63049626d11": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "3a46a46bc8124a92b27aef43cbc009b6": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "4ad6bc0cca62446d8faf19a341bfa86f": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "6437c99289f947449f7d2964288973e5": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "e2f7dea8fc744537b42d0f1a85a73eb4": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "1377d2160344430da8f29a50d113a288": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "0c0b30e126724f9282ac5acbcb4581db": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "895efd0b6d9f4b319159703d965d1966": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_dece6dff65394a5f93585c73359d4dad", - "IPY_MODEL_1030c0848635497681cc9ff0c344fb1a", - "IPY_MODEL_fa6ecaab432347de8427b9b5ac3d4524" - ], - "layout": "IPY_MODEL_5effefa8e3764e3aaff57fe0197a7c96" - } - }, - "dece6dff65394a5f93585c73359d4dad": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_1756eceba2c34c1ca182b7db465e95ce", - "placeholder": "​", - "style": "IPY_MODEL_0fd62e56e0bb41a996c04e63381d2a29", - "value": "config_sentence_transformers.json: 100%" - } - }, - "1030c0848635497681cc9ff0c344fb1a": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_29badfc2eb0345d38d7cfc6c7f8bb1a8", - "max": 116, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_e64cedb4560a43d8a43f36002087ac30", - "value": 116 - } - }, - "fa6ecaab432347de8427b9b5ac3d4524": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_45aadb26b382460eb5b6b147509fb75a", - "placeholder": "​", - "style": "IPY_MODEL_130f2f5840764e8dbd573cc8a6ea6f5f", - "value": " 116/116 [00:00<00:00, 3.35kB/s]" - } - }, - "5effefa8e3764e3aaff57fe0197a7c96": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "1756eceba2c34c1ca182b7db465e95ce": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "0fd62e56e0bb41a996c04e63381d2a29": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "29badfc2eb0345d38d7cfc6c7f8bb1a8": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "e64cedb4560a43d8a43f36002087ac30": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "45aadb26b382460eb5b6b147509fb75a": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "130f2f5840764e8dbd573cc8a6ea6f5f": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "9ee45247ec144bb3aafe4208f316063f": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_da330e0999cb4c3c91a1cb1026304568", - "IPY_MODEL_ff58a5381fb74cb1b9efc10f5c2738d6", - "IPY_MODEL_18ed62b1d4594ed9a2651fa5df046efc" - ], - "layout": "IPY_MODEL_4004cda1d84949f5a380536f8a9d0274" - } - }, - "da330e0999cb4c3c91a1cb1026304568": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_54bddcf41c5641b7a56c981aadb62ef1", - "placeholder": "​", - "style": "IPY_MODEL_a9a0d8415d9d4e98a3f02ae8ec1053da", - "value": "README.md: 100%" - } - }, - "ff58a5381fb74cb1b9efc10f5c2738d6": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_cceff1126242494bab432205c7ac7345", - "max": 10659, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_e6e53c439dab4639adc1c3c873602476", - "value": 10659 - } - }, - "18ed62b1d4594ed9a2651fa5df046efc": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_95db8eab3f964edf99038ad53f41fabc", - "placeholder": "​", - "style": "IPY_MODEL_52f1d69c6cd04816b6f34657893ae32b", - "value": " 10.7k/10.7k [00:00<00:00, 223kB/s]" - } - }, - "4004cda1d84949f5a380536f8a9d0274": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "54bddcf41c5641b7a56c981aadb62ef1": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "a9a0d8415d9d4e98a3f02ae8ec1053da": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "cceff1126242494bab432205c7ac7345": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "e6e53c439dab4639adc1c3c873602476": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "95db8eab3f964edf99038ad53f41fabc": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "52f1d69c6cd04816b6f34657893ae32b": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "b79a1dfcf2904bcba332569dbf351f34": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_7363b1a9a1b54a57bf15357e897128fd", - "IPY_MODEL_3ac596104cdc4439b3980f7ce66ad080", - "IPY_MODEL_5c9ec25994914acd8e13866b3eb943e1" - ], - "layout": "IPY_MODEL_38a958036c6e4155815a8169f1be1e53" - } - }, - "7363b1a9a1b54a57bf15357e897128fd": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_cf5113a647ce45c4a3a523361aa3b5af", - "placeholder": "​", - "style": "IPY_MODEL_da8c20a65ba541bda058614849d5cfe2", - "value": "sentence_bert_config.json: 100%" - } - }, - "3ac596104cdc4439b3980f7ce66ad080": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_40e9f20d74374b0e82c653caa0559d04", - "max": 53, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_f46cfc9237e64db6be2ec6529b61ec88", - "value": 53 - } - }, - "5c9ec25994914acd8e13866b3eb943e1": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_dc04575da46540d4ad3a708e58f0de6a", - "placeholder": "​", - "style": "IPY_MODEL_24c0be775e474517a7be49d187822bd0", - "value": " 53.0/53.0 [00:00<00:00, 3.84kB/s]" - } - }, - "38a958036c6e4155815a8169f1be1e53": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "cf5113a647ce45c4a3a523361aa3b5af": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "da8c20a65ba541bda058614849d5cfe2": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "40e9f20d74374b0e82c653caa0559d04": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "f46cfc9237e64db6be2ec6529b61ec88": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "dc04575da46540d4ad3a708e58f0de6a": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "24c0be775e474517a7be49d187822bd0": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "111184729957441d9d1f3d404bd82757": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_be060f9d7a664c17a80510f447c0bee3", - "IPY_MODEL_228445132e5f4b2ca793f4beeeca4426", - "IPY_MODEL_b96a2e34a2af435b9705550fe564591d" - ], - "layout": "IPY_MODEL_1f1cdac013af4559889f15eebac5256a" - } - }, - "be060f9d7a664c17a80510f447c0bee3": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_834ae2d249b94be6bbe5349509536a4b", - "placeholder": "​", - "style": "IPY_MODEL_509863a58de74b07b813aa83ffa4a507", - "value": "config.json: 100%" - } - }, - "228445132e5f4b2ca793f4beeeca4426": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_48a5b775a4324da791603b83d61be7d1", - "max": 612, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_02b60dad91c7482ba70cf8bb954bc4eb", - "value": 612 - } - }, - "b96a2e34a2af435b9705550fe564591d": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_2bfb0fb5506d4285918a9c94af9ab5d1", - "placeholder": "​", - "style": "IPY_MODEL_0f699b0f99484a8ba2eb17bb1d621c5a", - "value": " 612/612 [00:00<00:00, 47.5kB/s]" - } - }, - "1f1cdac013af4559889f15eebac5256a": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "834ae2d249b94be6bbe5349509536a4b": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "509863a58de74b07b813aa83ffa4a507": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "48a5b775a4324da791603b83d61be7d1": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "02b60dad91c7482ba70cf8bb954bc4eb": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "2bfb0fb5506d4285918a9c94af9ab5d1": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "0f699b0f99484a8ba2eb17bb1d621c5a": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "c6f34317390e4f90b16235f2ae84a981": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_3da95c8814f34472a181ce7687f9e15e", - "IPY_MODEL_4d1c2de4c1354ef0b84c54c447141707", - "IPY_MODEL_31ab98e0e375416b83b36a98d4958f57" - ], - "layout": "IPY_MODEL_8b9ebe06b4e045a29269128ec97d9f62" - } - }, - "3da95c8814f34472a181ce7687f9e15e": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_53a46fe254924e78876db6dd2e1b7123", - "placeholder": "​", - "style": "IPY_MODEL_f2ce01983f0a4f12b318e6d29f1dd4a1", - "value": "model.safetensors: 100%" - } - }, - "4d1c2de4c1354ef0b84c54c447141707": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_1b7af9f7204547b8b4a718a780af0ded", - "max": 90868376, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_a4bb5a59d1324585b0a34c9bb2820b7f", - "value": 90868376 - } - }, - "31ab98e0e375416b83b36a98d4958f57": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_90c2e0e012a94521b9f5cb24924771d8", - "placeholder": "​", - "style": "IPY_MODEL_2563a4677dde47d0a2f7fba5c5dde358", - "value": " 90.9M/90.9M [00:00<00:00, 223MB/s]" - } - }, - "8b9ebe06b4e045a29269128ec97d9f62": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "53a46fe254924e78876db6dd2e1b7123": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "f2ce01983f0a4f12b318e6d29f1dd4a1": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "1b7af9f7204547b8b4a718a780af0ded": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "a4bb5a59d1324585b0a34c9bb2820b7f": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "90c2e0e012a94521b9f5cb24924771d8": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "2563a4677dde47d0a2f7fba5c5dde358": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "5023c2b8cf9846069d116237826fed7f": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_960c2f44166b4ac7910af6512832186f", - "IPY_MODEL_309ea9620a674088a5207206d9a52d54", - "IPY_MODEL_1c86d856083c4ef99976849c7a1c9100" - ], - "layout": "IPY_MODEL_5d9bf2102da143c1b9e1483e05add4e5" - } - }, - "960c2f44166b4ac7910af6512832186f": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_85569eaf3ae3488b808131cd460f6514", - "placeholder": "​", - "style": "IPY_MODEL_3015bc3ce98a4221a9dd3be92481435d", - "value": "tokenizer_config.json: 100%" - } - }, - "309ea9620a674088a5207206d9a52d54": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_4d7b0983b97f48b2a333d5b2a4ec50a8", - "max": 350, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_e834a64e49534c3586cb77f4ec5eab2d", - "value": 350 - } - }, - "1c86d856083c4ef99976849c7a1c9100": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_67f82b82ebb74d0fb3c68b9c8c57d690", - "placeholder": "​", - "style": "IPY_MODEL_b710cb57f19d4490a740c060e8a83b90", - "value": " 350/350 [00:00<00:00, 26.0kB/s]" - } - }, - "5d9bf2102da143c1b9e1483e05add4e5": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "85569eaf3ae3488b808131cd460f6514": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "3015bc3ce98a4221a9dd3be92481435d": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "4d7b0983b97f48b2a333d5b2a4ec50a8": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "e834a64e49534c3586cb77f4ec5eab2d": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "67f82b82ebb74d0fb3c68b9c8c57d690": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "b710cb57f19d4490a740c060e8a83b90": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "713c09d1275a43b0af7c2ae8e126517f": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_b62fe08114f549ea99808e8df95c7cad", - "IPY_MODEL_af722d177320422e97c679b24cb754f6", - "IPY_MODEL_487477e023b64947bf42f83dc6275ef1" - ], - "layout": "IPY_MODEL_bcf0d3af3bc0439e97023937852941e9" - } - }, - "b62fe08114f549ea99808e8df95c7cad": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_d83a1e1e678e4efd83115f9aee0ffc8d", - "placeholder": "​", - "style": "IPY_MODEL_f210583576594e759387fc704695ad09", - "value": "vocab.txt: 100%" - } - }, - "af722d177320422e97c679b24cb754f6": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_91e103573c034ceda689047c61294b17", - "max": 231508, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_b9eac61fb55342f4bf9834f321899836", - "value": 231508 - } - }, - "487477e023b64947bf42f83dc6275ef1": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_a92a7bce961e4291b126fda3c540636b", - "placeholder": "​", - "style": "IPY_MODEL_01b3e7803d1946118d27acda0c067da2", - "value": " 232k/232k [00:00<00:00, 550kB/s]" - } - }, - "bcf0d3af3bc0439e97023937852941e9": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "d83a1e1e678e4efd83115f9aee0ffc8d": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "f210583576594e759387fc704695ad09": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "91e103573c034ceda689047c61294b17": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "b9eac61fb55342f4bf9834f321899836": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "a92a7bce961e4291b126fda3c540636b": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "01b3e7803d1946118d27acda0c067da2": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "f097b32928f246de9b01fea6f9b092f7": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_35e10db3906248ffa8ab955d2f53bd75", - "IPY_MODEL_80e884cae6ea42eaa37f028120963355", - "IPY_MODEL_25821e7aef4e481bbdf3b4698ce3c277" - ], - "layout": "IPY_MODEL_916190b4615e4c5c9f3e55c0804a3502" - } - }, - "35e10db3906248ffa8ab955d2f53bd75": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_1f1dc0d20cae46feb372203aea6458a0", - "placeholder": "​", - "style": "IPY_MODEL_43feace0290a47c0b06c3a1c08cc70a9", - "value": "tokenizer.json: 100%" - } - }, - "80e884cae6ea42eaa37f028120963355": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_9f185162847f4cb2828af81c92116582", - "max": 466247, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_3a649adc22694036b35bab04ff03d338", - "value": 466247 - } - }, - "25821e7aef4e481bbdf3b4698ce3c277": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_7daef1502e2a4140ac021b3b3a6aa12d", - "placeholder": "​", - "style": "IPY_MODEL_1307ef0325bb433d8a1bcc653c7fb291", - "value": " 466k/466k [00:00<00:00, 2.16MB/s]" - } - }, - "916190b4615e4c5c9f3e55c0804a3502": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "1f1dc0d20cae46feb372203aea6458a0": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "43feace0290a47c0b06c3a1c08cc70a9": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "9f185162847f4cb2828af81c92116582": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "3a649adc22694036b35bab04ff03d338": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "7daef1502e2a4140ac021b3b3a6aa12d": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "1307ef0325bb433d8a1bcc653c7fb291": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "f01d7a1404a943a08c84adce14a262c7": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_f15cdedf8e7b4a44993644a5ff070e78", - "IPY_MODEL_b7f9a3c97f2043f380bdc1827961c649", - "IPY_MODEL_0b64892a98d14a3b85b128df77d8e7d6" - ], - "layout": "IPY_MODEL_8de1cba3a7c0422eb2a21e3f8b2059c7" - } - }, - "f15cdedf8e7b4a44993644a5ff070e78": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_a0639d5360044f97ac5b9374c735ff4b", - "placeholder": "​", - "style": "IPY_MODEL_9b11eaf2d50a447384b75eb7f73829eb", - "value": "special_tokens_map.json: 100%" - } - }, - "b7f9a3c97f2043f380bdc1827961c649": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_8ab411217bfd486ca3fb8b885fff4690", - "max": 112, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_c80ea8c54211427087712b5500e26edf", - "value": 112 - } - }, - "0b64892a98d14a3b85b128df77d8e7d6": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_542aa4a847cf4a66a4b3fc93c241363b", - "placeholder": "​", - "style": "IPY_MODEL_8c0d69b735c94b719160d39256c643cc", - "value": " 112/112 [00:00<00:00, 6.51kB/s]" - } - }, - "8de1cba3a7c0422eb2a21e3f8b2059c7": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "a0639d5360044f97ac5b9374c735ff4b": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "9b11eaf2d50a447384b75eb7f73829eb": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "8ab411217bfd486ca3fb8b885fff4690": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "c80ea8c54211427087712b5500e26edf": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "542aa4a847cf4a66a4b3fc93c241363b": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "8c0d69b735c94b719160d39256c643cc": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "3c868641db934c67a44e1d26e1a17756": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_a72d01788b484bbeb4375aac3ceadf34", - "IPY_MODEL_366add01dc734455a384460c97491215", - "IPY_MODEL_70accb92e645435b8f1e0c48538f7473" - ], - "layout": "IPY_MODEL_628848757fcf443e806a8f25013cc2b5" - } - }, - "a72d01788b484bbeb4375aac3ceadf34": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_ebf411690c844daf89b87c120e3cb67e", - "placeholder": "​", - "style": "IPY_MODEL_79b9fb75dc1d486c9fc881a90b6f1060", - "value": "1_Pooling/config.json: 100%" - } - }, - "366add01dc734455a384460c97491215": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_0f3bbf28fbed4e97b660bbf3c66a214a", - "max": 190, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_a4b2220ed47f4f85b3f991c92de98964", - "value": 190 - } - }, - "70accb92e645435b8f1e0c48538f7473": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_b6a505e6c863409db1b906423f99125a", - "placeholder": "​", - "style": "IPY_MODEL_d9560d20106a42ec904e7e315f99ff01", - "value": " 190/190 [00:00<00:00, 9.18kB/s]" - } - }, - "628848757fcf443e806a8f25013cc2b5": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "ebf411690c844daf89b87c120e3cb67e": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "79b9fb75dc1d486c9fc881a90b6f1060": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "0f3bbf28fbed4e97b660bbf3c66a214a": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "a4b2220ed47f4f85b3f991c92de98964": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "b6a505e6c863409db1b906423f99125a": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "d9560d20106a42ec904e7e315f99ff01": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "edc4d84302f746d39a43e8107af6b67b": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_980292182c7144e194604c13ac544a26", - "IPY_MODEL_8dee873065a047799a04e49ab791e449", - "IPY_MODEL_29683ef34d5646c687118a2a0cdec6d4" - ], - "layout": "IPY_MODEL_3ec694106303491ea112a257309bc69c" - } - }, - "980292182c7144e194604c13ac544a26": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_288c9da81b3c4d80a4959753da973f58", - "placeholder": "​", - "style": "IPY_MODEL_cf453a1ed54645aba656f9a3f1461e69", - "value": "Batches: 100%" - } - }, - "8dee873065a047799a04e49ab791e449": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_ec747bd7c37c45298896c513634cd59a", - "max": 1, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_5a620017a5384af1a056de687b2670db", - "value": 1 - } - }, - "29683ef34d5646c687118a2a0cdec6d4": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_8d370762fafd4d7887ff68ea8279d083", - "placeholder": "​", - "style": "IPY_MODEL_b6a0eb553b024a71b737ff47ca8f7633", - "value": " 1/1 [00:01<00:00,  1.24s/it]" - } - }, - "3ec694106303491ea112a257309bc69c": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "288c9da81b3c4d80a4959753da973f58": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "cf453a1ed54645aba656f9a3f1461e69": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "ec747bd7c37c45298896c513634cd59a": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "5a620017a5384af1a056de687b2670db": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "8d370762fafd4d7887ff68ea8279d083": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "b6a0eb553b024a71b737ff47ca8f7633": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "2eff72cbd9bb4f1ca77213602caa9417": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_e82b5196209f4b9f919c7abb402a4504", - "IPY_MODEL_fe34706489c14253a5015ff6332ec4e0", - "IPY_MODEL_2574b07e4af24715aa89d048cc84e358" - ], - "layout": "IPY_MODEL_10bc8be68b5545fd8609824b02499ebf" - } - }, - "e82b5196209f4b9f919c7abb402a4504": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_d2473b7a6c5b4483981516af2fc59bde", - "placeholder": "​", - "style": "IPY_MODEL_4282ee7d947e426ba863df9970e82f3f", - "value": "Batches: 100%" - } - }, - "fe34706489c14253a5015ff6332ec4e0": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_cfe6be8fd8254bc084a81b1d06e86ae1", - "max": 1, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_1817f6732a5f44c7adc75a644b1acef2", - "value": 1 - } - }, - "2574b07e4af24715aa89d048cc84e358": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_7551b282ef3a4387a801637de2d5c76e", - "placeholder": "​", - "style": "IPY_MODEL_69e5263c812c4542a9e5c31fefaa37fe", - "value": " 1/1 [00:00<00:00, 15.08it/s]" - } - }, - "10bc8be68b5545fd8609824b02499ebf": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "d2473b7a6c5b4483981516af2fc59bde": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "4282ee7d947e426ba863df9970e82f3f": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "cfe6be8fd8254bc084a81b1d06e86ae1": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "1817f6732a5f44c7adc75a644b1acef2": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "7551b282ef3a4387a801637de2d5c76e": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "69e5263c812c4542a9e5c31fefaa37fe": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "7cc356ed20e94401b72a0e138ad0f5df": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_acd39276db17439798a97abc56460b0f", - "IPY_MODEL_bda474c3b8184597a6a9bc6da0672a50", - "IPY_MODEL_20a66f9de4ed41c7ac9a8e817898ed9e" - ], - "layout": "IPY_MODEL_e662ba10fbae49d9b66172125dfc0717" - } - }, - "acd39276db17439798a97abc56460b0f": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_d452b32c54e14e41a17fd7d51862ba8e", - "placeholder": "​", - "style": "IPY_MODEL_d1f8f4568a444248b69022d58e3f1af0", - "value": "Batches: 100%" - } - }, - "bda474c3b8184597a6a9bc6da0672a50": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_0c2e30d78c234b1b8098d879442d3bac", - "max": 1, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_9bb8bf12010f42b2b17c10c7ccaa7bf8", - "value": 1 - } - }, - "20a66f9de4ed41c7ac9a8e817898ed9e": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_2b2046db907349798e3ae774c15b25d2", - "placeholder": "​", - "style": "IPY_MODEL_3c18f449359f422f950543bd976fe323", - "value": " 1/1 [00:00<00:00, 18.91it/s]" - } - }, - "e662ba10fbae49d9b66172125dfc0717": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "d452b32c54e14e41a17fd7d51862ba8e": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "d1f8f4568a444248b69022d58e3f1af0": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "0c2e30d78c234b1b8098d879442d3bac": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "9bb8bf12010f42b2b17c10c7ccaa7bf8": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "2b2046db907349798e3ae774c15b25d2": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "3c18f449359f422f950543bd976fe323": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "472b1acc4c5a4c48b2ec62be42d1830c": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_44e34588d6854737b0fb14b4b6a62a95", - "IPY_MODEL_03402ad03418435ca7a550e3246cd300", - "IPY_MODEL_811f115733b14ab4b242a8b11526016c" - ], - "layout": "IPY_MODEL_e61fdef1dc4b4d809168c0b441b0e6ac" - } - }, - "44e34588d6854737b0fb14b4b6a62a95": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_631c9a95127244c79875c829a7637df6", - "placeholder": "​", - "style": "IPY_MODEL_d25492ad867141bfa8d957d2464b8639", - "value": "Batches: 100%" - } - }, - "03402ad03418435ca7a550e3246cd300": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_9df914248c214597bed7d7980c7a0afe", - "max": 1, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_4709067f3f554b93b3ef35e3f58cbf85", - "value": 1 - } - }, - "811f115733b14ab4b242a8b11526016c": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_02baf670942347d69c290452de8641e4", - "placeholder": "​", - "style": "IPY_MODEL_7611cfc7965649ba88ca57c1a9f9ccf3", - "value": " 1/1 [00:00<00:00, 13.00it/s]" - } - }, - "e61fdef1dc4b4d809168c0b441b0e6ac": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "631c9a95127244c79875c829a7637df6": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "d25492ad867141bfa8d957d2464b8639": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "9df914248c214597bed7d7980c7a0afe": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "4709067f3f554b93b3ef35e3f58cbf85": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "02baf670942347d69c290452de8641e4": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "7611cfc7965649ba88ca57c1a9f9ccf3": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "15ae23892b634a9f821a8fcee14e500b": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HBoxModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_b28d46c2ecdd46b9b3f2da871afbf1cb", - "IPY_MODEL_4b83e3caa8ec47169dca04ee9599adeb", - "IPY_MODEL_c83c23161674484e81f0db9856c23eb6" - ], - "layout": "IPY_MODEL_3ded85d9c34246e88f8ce693eb8025e5" - } - }, - "b28d46c2ecdd46b9b3f2da871afbf1cb": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_0ac8e976a32c4f5989392b8088546e00", - "placeholder": "​", - "style": "IPY_MODEL_ed4b0035752546cc81688a7a77ba27c0", - "value": "Batches: 100%" - } - }, - "4b83e3caa8ec47169dca04ee9599adeb": { - "model_module": "@jupyter-widgets/controls", - "model_name": "FloatProgressModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_269b1ad9dc7b4ebb94d7364c75f3f324", - "max": 1, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_2256ddab0ae1408abb10ba211a08f794", - "value": 1 - } - }, - "c83c23161674484e81f0db9856c23eb6": { - "model_module": "@jupyter-widgets/controls", - "model_name": "HTMLModel", - "model_module_version": "1.5.0", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_42335bcbc6ee40a79d36c5159cc7da06", - "placeholder": "​", - "style": "IPY_MODEL_cf694e1b797246b096ae588973dc985f", - "value": " 1/1 [00:00<00:00, 14.00it/s]" - } - }, - "3ded85d9c34246e88f8ce693eb8025e5": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "0ac8e976a32c4f5989392b8088546e00": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "ed4b0035752546cc81688a7a77ba27c0": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "269b1ad9dc7b4ebb94d7364c75f3f324": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "2256ddab0ae1408abb10ba211a08f794": { - "model_module": "@jupyter-widgets/controls", - "model_name": "ProgressStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "42335bcbc6ee40a79d36c5159cc7da06": { - "model_module": "@jupyter-widgets/base", - "model_name": "LayoutModel", - "model_module_version": "1.2.0", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "cf694e1b797246b096ae588973dc985f": { - "model_module": "@jupyter-widgets/controls", - "model_name": "DescriptionStyleModel", - "model_module_version": "1.5.0", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - } - } - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/docs/openapi_generator/pyopenapi/generator.py b/docs/openapi_generator/pyopenapi/generator.py index 23465257a..25b08f071 100644 --- a/docs/openapi_generator/pyopenapi/generator.py +++ b/docs/openapi_generator/pyopenapi/generator.py @@ -537,7 +537,6 @@ class Generator: success_type_descriptions = { item: doc_string.short_description for item, doc_string in success_type_docstring.items() - if doc_string.short_description } else: # use return type as a single response type @@ -596,6 +595,7 @@ class Generator: ) responses.update(response_builder.build_response(response_options)) + assert len(responses.keys()) > 0, f"No responses found for {op.name}" if op.event_type is not None: builder = ContentBuilder(self.schema_builder) callbacks = { diff --git a/docs/openapi_generator/pyopenapi/operations.py b/docs/openapi_generator/pyopenapi/operations.py index cc3a06b7b..abeb16936 100644 --- a/docs/openapi_generator/pyopenapi/operations.py +++ b/docs/openapi_generator/pyopenapi/operations.py @@ -8,7 +8,6 @@ import collections.abc import enum import inspect import typing -import uuid from dataclasses import dataclass from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, Tuple, Union @@ -16,12 +15,7 @@ from llama_stack.apis.version import LLAMA_STACK_API_VERSION from termcolor import colored -from ..strong_typing.inspection import ( - get_signature, - is_type_enum, - is_type_optional, - unwrap_optional_type, -) +from ..strong_typing.inspection import get_signature def split_prefix( @@ -113,9 +107,6 @@ class EndpointOperation: def get_route(self) -> str: if self.route is not None: - assert ( - "_" not in self.route - ), f"route should not contain underscores: {self.route}" return "/".join(["", LLAMA_STACK_API_VERSION, self.route.lstrip("/")]) route_parts = ["", LLAMA_STACK_API_VERSION, self.name] @@ -181,10 +172,16 @@ def _get_endpoint_functions( def _get_defining_class(member_fn: str, derived_cls: type) -> type: "Find the class in which a member function is first defined in a class inheritance hierarchy." + # This import must be dynamic here + from llama_stack.apis.tools import RAGToolRuntime, ToolRuntime + # iterate in reverse member resolution order to find most specific class first for cls in reversed(inspect.getmro(derived_cls)): for name, _ in inspect.getmembers(cls, inspect.isfunction): if name == member_fn: + # HACK ALERT + if cls == RAGToolRuntime: + return ToolRuntime return cls raise ValidationError( @@ -265,42 +262,16 @@ def get_endpoint_operations( f"parameter '{param_name}' in function '{func_name}' has no type annotation" ) - if is_type_optional(param_type): - inner_type: type = unwrap_optional_type(param_type) - else: - inner_type = param_type - - if prefix == "get" and ( - inner_type is bool - or inner_type is int - or inner_type is float - or inner_type is str - or inner_type is uuid.UUID - or is_type_enum(inner_type) - ): - if parameter.kind == inspect.Parameter.POSITIONAL_ONLY: - if route_params is not None and param_name not in route_params: - raise ValidationError( - f"positional parameter '{param_name}' absent from user-defined route '{route}' for function '{func_name}'" - ) - - # simple type maps to route path element, e.g. /study/{uuid}/{version} + if prefix in ["get", "delete"]: + if route_params is not None and param_name in route_params: path_params.append((param_name, param_type)) else: - if route_params is not None and param_name in route_params: - raise ValidationError( - f"query parameter '{param_name}' found in user-defined route '{route}' for function '{func_name}'" - ) - - # simple type maps to key=value pair in query string query_params.append((param_name, param_type)) else: if route_params is not None and param_name in route_params: - raise ValidationError( - f"user-defined route '{route}' for function '{func_name}' has parameter '{param_name}' of composite type: {param_type}" - ) - - request_params.append((param_name, param_type)) + path_params.append((param_name, param_type)) + else: + request_params.append((param_name, param_type)) # check if function has explicit return type if signature.return_annotation is inspect.Signature.empty: @@ -335,19 +306,18 @@ def get_endpoint_operations( response_type = process_type(return_type) - # set HTTP request method based on type of request and presence of payload - if not request_params: if prefix in ["delete", "remove"]: http_method = HTTPMethod.DELETE - else: + elif prefix == "post": + http_method = HTTPMethod.POST + elif prefix == "get": http_method = HTTPMethod.GET - else: - if prefix == "set": + elif prefix == "set": http_method = HTTPMethod.PUT elif prefix == "update": http_method = HTTPMethod.PATCH else: - http_method = HTTPMethod.POST + raise ValidationError(f"unknown prefix {prefix}") result.append( EndpointOperation( diff --git a/docs/openapi_generator/strong_typing/classdef.py b/docs/openapi_generator/strong_typing/classdef.py index c8e6781fd..788ecc7e0 100644 --- a/docs/openapi_generator/strong_typing/classdef.py +++ b/docs/openapi_generator/strong_typing/classdef.py @@ -125,6 +125,7 @@ class JsonSchemaAnyOf(JsonSchemaNode): @dataclass class JsonSchemaOneOf(JsonSchemaNode): oneOf: List["JsonSchemaAny"] + discriminator: Optional[str] JsonSchemaAny = Union[ diff --git a/docs/openapi_generator/strong_typing/inspection.py b/docs/openapi_generator/strong_typing/inspection.py index c5e7899fa..41804f12c 100644 --- a/docs/openapi_generator/strong_typing/inspection.py +++ b/docs/openapi_generator/strong_typing/inspection.py @@ -342,7 +342,6 @@ def is_type_union(typ: object) -> bool: "True if the type annotation corresponds to a union type (e.g. `Union[T1,T2,T3]`)." typ = unwrap_annotated_type(typ) - if _is_union_like(typ): args = typing.get_args(typ) return len(args) > 2 or type(None) not in args diff --git a/docs/openapi_generator/strong_typing/schema.py b/docs/openapi_generator/strong_typing/schema.py index 42feeee5a..5aa41b63f 100644 --- a/docs/openapi_generator/strong_typing/schema.py +++ b/docs/openapi_generator/strong_typing/schema.py @@ -36,6 +36,7 @@ from typing import ( ) import jsonschema +from typing_extensions import Annotated from . import docstring from .auxiliary import ( @@ -329,7 +330,6 @@ class JsonSchemaGenerator: if metadata is not None: # type is Annotated[T, ...] typ = typing.get_args(data_type)[0] - schema = self._simple_type_to_schema(typ) if schema is not None: # recognize well-known auxiliary types @@ -446,12 +446,20 @@ class JsonSchemaGenerator: ], } elif origin_type is Union: - return { + discriminator = None + if typing.get_origin(data_type) is Annotated: + discriminator = typing.get_args(data_type)[1].discriminator + ret = { "oneOf": [ self.type_to_schema(union_type) for union_type in typing.get_args(typ) ] } + if discriminator: + ret["discriminator"] = { + "propertyName": discriminator, + } + return ret elif origin_type is Literal: (literal_value,) = typing.get_args(typ) # unpack value of literal type schema = self.type_to_schema(type(literal_value)) diff --git a/docs/resources/llama-stack-spec.html b/docs/resources/llama-stack-spec.html index 5ed8701a4..f6024c586 100644 --- a/docs/resources/llama-stack-spec.html +++ b/docs/resources/llama-stack-spec.html @@ -20,7 +20,7 @@ "openapi": "3.1.0", "info": { "title": "Llama Stack Specification", - "version": "alpha", + "version": "v1", "description": "This is the specification of the Llama Stack that provides\n a set of endpoints and their corresponding interfaces that are tailored to\n best leverage Llama Models." }, "servers": [ @@ -29,1065 +29,7 @@ } ], "paths": { - "/alpha/datasetio/append-rows": { - "post": { - "responses": { - "200": { - "description": "OK" - } - }, - "tags": [ - "DatasetIO" - ], - "parameters": [ - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AppendRowsRequest" - } - } - }, - "required": true - } - } - }, - "/alpha/batch-inference/chat-completion": { - "post": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/BatchChatCompletionResponse" - } - } - } - } - }, - "tags": [ - "BatchInference (Coming Soon)" - ], - "parameters": [ - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/BatchChatCompletionRequest" - } - } - }, - "required": true - } - } - }, - "/alpha/batch-inference/completion": { - "post": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/BatchCompletionResponse" - } - } - } - } - }, - "tags": [ - "BatchInference (Coming Soon)" - ], - "parameters": [ - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/BatchCompletionRequest" - } - } - }, - "required": true - } - } - }, - "/alpha/post-training/job/cancel": { - "post": { - "responses": { - "200": { - "description": "OK" - } - }, - "tags": [ - "PostTraining (Coming Soon)" - ], - "parameters": [ - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CancelTrainingJobRequest" - } - } - }, - "required": true - } - } - }, - "/alpha/inference/chat-completion": { - "post": { - "responses": { - "200": { - "description": "Chat completion response. **OR** SSE-stream of these events.", - "content": { - "text/event-stream": { - "schema": { - "oneOf": [ - { - "$ref": "#/components/schemas/ChatCompletionResponse" - }, - { - "$ref": "#/components/schemas/ChatCompletionResponseStreamChunk" - } - ] - } - } - } - } - }, - "tags": [ - "Inference" - ], - "parameters": [ - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ChatCompletionRequest" - } - } - }, - "required": true - } - } - }, - "/alpha/inference/completion": { - "post": { - "responses": { - "200": { - "description": "Completion response. **OR** streamed completion response.", - "content": { - "text/event-stream": { - "schema": { - "oneOf": [ - { - "$ref": "#/components/schemas/CompletionResponse" - }, - { - "$ref": "#/components/schemas/CompletionResponseStreamChunk" - } - ] - } - } - } - } - }, - "tags": [ - "Inference" - ], - "parameters": [ - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CompletionRequest" - } - } - }, - "required": true - } - } - }, - "/alpha/agents/create": { - "post": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AgentCreateResponse" - } - } - } - } - }, - "tags": [ - "Agents" - ], - "parameters": [ - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CreateAgentRequest" - } - } - }, - "required": true - } - } - }, - "/alpha/agents/session/create": { - "post": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AgentSessionCreateResponse" - } - } - } - } - }, - "tags": [ - "Agents" - ], - "parameters": [ - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CreateAgentSessionRequest" - } - } - }, - "required": true - } - } - }, - "/alpha/agents/turn/create": { - "post": { - "responses": { - "200": { - "description": "A single turn in an interaction with an Agentic System. **OR** streamed agent turn completion response.", - "content": { - "text/event-stream": { - "schema": { - "oneOf": [ - { - "$ref": "#/components/schemas/Turn" - }, - { - "$ref": "#/components/schemas/AgentTurnResponseStreamChunk" - } - ] - } - } - } - } - }, - "tags": [ - "Agents" - ], - "parameters": [ - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CreateAgentTurnRequest" - } - } - }, - "required": true - } - } - }, - "/alpha/agents/delete": { - "post": { - "responses": { - "200": { - "description": "OK" - } - }, - "tags": [ - "Agents" - ], - "parameters": [ - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/DeleteAgentsRequest" - } - } - }, - "required": true - } - } - }, - "/alpha/agents/session/delete": { - "post": { - "responses": { - "200": { - "description": "OK" - } - }, - "tags": [ - "Agents" - ], - "parameters": [ - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/DeleteAgentsSessionRequest" - } - } - }, - "required": true - } - } - }, - "/alpha/inference/embeddings": { - "post": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/EmbeddingsResponse" - } - } - } - } - }, - "tags": [ - "Inference" - ], - "parameters": [ - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/EmbeddingsRequest" - } - } - }, - "required": true - } - } - }, - "/alpha/eval/evaluate-rows": { - "post": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/EvaluateResponse" - } - } - } - } - }, - "tags": [ - "Eval" - ], - "parameters": [ - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/EvaluateRowsRequest" - } - } - }, - "required": true - } - } - }, - "/alpha/agents/session/get": { - "post": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Session" - } - } - } - } - }, - "tags": [ - "Agents" - ], - "parameters": [ - { - "name": "agent_id", - "in": "query", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "session_id", - "in": "query", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/GetAgentsSessionRequest" - } - } - }, - "required": true - } - } - }, - "/alpha/agents/step/get": { - "get": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AgentStepResponse" - } - } - } - } - }, - "tags": [ - "Agents" - ], - "parameters": [ - { - "name": "agent_id", - "in": "query", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "session_id", - "in": "query", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "turn_id", - "in": "query", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "step_id", - "in": "query", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ] - } - }, - "/alpha/agents/turn/get": { - "get": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Turn" - } - } - } - } - }, - "tags": [ - "Agents" - ], - "parameters": [ - { - "name": "agent_id", - "in": "query", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "session_id", - "in": "query", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "turn_id", - "in": "query", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ] - } - }, - "/alpha/datasets/get": { - "get": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/json": { - "schema": { - "oneOf": [ - { - "$ref": "#/components/schemas/Dataset" - }, - { - "type": "null" - } - ] - } - } - } - } - }, - "tags": [ - "Datasets" - ], - "parameters": [ - { - "name": "dataset_id", - "in": "query", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ] - } - }, - "/alpha/eval-tasks/get": { - "get": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/json": { - "schema": { - "oneOf": [ - { - "$ref": "#/components/schemas/EvalTask" - }, - { - "type": "null" - } - ] - } - } - } - } - }, - "tags": [ - "EvalTasks" - ], - "parameters": [ - { - "name": "name", - "in": "query", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ] - } - }, - "/alpha/memory-banks/get": { - "get": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/json": { - "schema": { - "oneOf": [ - { - "oneOf": [ - { - "$ref": "#/components/schemas/VectorMemoryBank" - }, - { - "$ref": "#/components/schemas/KeyValueMemoryBank" - }, - { - "$ref": "#/components/schemas/KeywordMemoryBank" - }, - { - "$ref": "#/components/schemas/GraphMemoryBank" - } - ] - }, - { - "type": "null" - } - ] - } - } - } - } - }, - "tags": [ - "MemoryBanks" - ], - "parameters": [ - { - "name": "memory_bank_id", - "in": "query", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ] - } - }, - "/alpha/models/get": { - "get": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/json": { - "schema": { - "oneOf": [ - { - "$ref": "#/components/schemas/Model" - }, - { - "type": "null" - } - ] - } - } - } - } - }, - "tags": [ - "Models" - ], - "parameters": [ - { - "name": "identifier", - "in": "query", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ] - } - }, - "/alpha/datasetio/get-rows-paginated": { + "/v1/datasetio/rows": { "get": { "responses": { "200": { @@ -1156,9 +98,1109 @@ } } ] + }, + "post": { + "responses": { + "200": { + "description": "OK" + } + }, + "tags": [ + "DatasetIO" + ], + "parameters": [ + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AppendRowsRequest" + } + } + }, + "required": true + } } }, - "/alpha/scoring-functions/get": { + "/v1/batch-inference/chat-completion": { + "post": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BatchChatCompletionResponse" + } + } + } + } + }, + "tags": [ + "BatchInference (Coming Soon)" + ], + "parameters": [ + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BatchChatCompletionRequest" + } + } + }, + "required": true + } + } + }, + "/v1/batch-inference/completion": { + "post": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BatchCompletionResponse" + } + } + } + } + }, + "tags": [ + "BatchInference (Coming Soon)" + ], + "parameters": [ + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BatchCompletionRequest" + } + } + }, + "required": true + } + } + }, + "/v1/post-training/job/cancel": { + "post": { + "responses": { + "200": { + "description": "OK" + } + }, + "tags": [ + "PostTraining (Coming Soon)" + ], + "parameters": [ + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CancelTrainingJobRequest" + } + } + }, + "required": true + } + } + }, + "/v1/inference/chat-completion": { + "post": { + "responses": { + "200": { + "description": "Chat completion response. **OR** SSE-stream of these events.", + "content": { + "text/event-stream": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/ChatCompletionResponse" + }, + { + "$ref": "#/components/schemas/ChatCompletionResponseStreamChunk" + } + ] + } + } + } + } + }, + "tags": [ + "Inference" + ], + "parameters": [ + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ChatCompletionRequest" + } + } + }, + "required": true + } + } + }, + "/v1/inference/completion": { + "post": { + "responses": { + "200": { + "description": "Completion response. **OR** streamed completion response.", + "content": { + "text/event-stream": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/CompletionResponse" + }, + { + "$ref": "#/components/schemas/CompletionResponseStreamChunk" + } + ] + } + } + } + } + }, + "tags": [ + "Inference" + ], + "parameters": [ + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CompletionRequest" + } + } + }, + "required": true + } + } + }, + "/v1/agents": { + "post": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentCreateResponse" + } + } + } + } + }, + "tags": [ + "Agents" + ], + "parameters": [ + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateAgentRequest" + } + } + }, + "required": true + } + } + }, + "/v1/agents/{agent_id}/session": { + "post": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentSessionCreateResponse" + } + } + } + } + }, + "tags": [ + "Agents" + ], + "parameters": [ + { + "name": "agent_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateAgentSessionRequest" + } + } + }, + "required": true + } + } + }, + "/v1/agents/{agent_id}/session/{session_id}/turn": { + "post": { + "responses": { + "200": { + "description": "A single turn in an interaction with an Agentic System. **OR** streamed agent turn completion response.", + "content": { + "text/event-stream": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/Turn" + }, + { + "$ref": "#/components/schemas/AgentTurnResponseStreamChunk" + } + ] + } + } + } + } + }, + "tags": [ + "Agents" + ], + "parameters": [ + { + "name": "agent_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "session_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateAgentTurnRequest" + } + } + }, + "required": true + } + } + }, + "/v1/agents/{agent_id}": { + "delete": { + "responses": { + "200": { + "description": "OK" + } + }, + "tags": [ + "Agents" + ], + "parameters": [ + { + "name": "agent_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ] + } + }, + "/v1/agents/{agent_id}/session/{session_id}": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Session" + } + } + } + } + }, + "tags": [ + "Agents" + ], + "parameters": [ + { + "name": "session_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "agent_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "turn_ids", + "in": "query", + "required": false, + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ] + }, + "delete": { + "responses": { + "200": { + "description": "OK" + } + }, + "tags": [ + "Agents" + ], + "parameters": [ + { + "name": "session_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "agent_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ] + } + }, + "/v1/inference/embeddings": { + "post": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EmbeddingsResponse" + } + } + } + } + }, + "tags": [ + "Inference" + ], + "parameters": [ + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EmbeddingsRequest" + } + } + }, + "required": true + } + } + }, + "/v1/eval/tasks/{task_id}/evaluations": { + "post": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EvaluateResponse" + } + } + } + } + }, + "tags": [ + "Eval" + ], + "parameters": [ + { + "name": "task_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EvaluateRowsRequest" + } + } + }, + "required": true + } + } + }, + "/v1/agents/{agent_id}/session/{session_id}/turn/{turn_id}/step/{step_id}": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AgentStepResponse" + } + } + } + } + }, + "tags": [ + "Agents" + ], + "parameters": [ + { + "name": "agent_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "session_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "turn_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "step_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ] + } + }, + "/v1/agents/{agent_id}/session/{session_id}/turn/{turn_id}": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Turn" + } + } + } + } + }, + "tags": [ + "Agents" + ], + "parameters": [ + { + "name": "agent_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "session_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "turn_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ] + } + }, + "/v1/datasets/{dataset_id}": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/Dataset" + }, + { + "type": "null" + } + ] + } + } + } + } + }, + "tags": [ + "Datasets" + ], + "parameters": [ + { + "name": "dataset_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ] + }, + "delete": { + "responses": { + "200": { + "description": "OK" + } + }, + "tags": [ + "Datasets" + ], + "parameters": [ + { + "name": "dataset_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ] + } + }, + "/v1/eval-tasks/{eval_task_id}": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/EvalTask" + }, + { + "type": "null" + } + ] + } + } + } + } + }, + "tags": [ + "EvalTasks" + ], + "parameters": [ + { + "name": "eval_task_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ] + } + }, + "/v1/models/{model_id}": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/Model" + }, + { + "type": "null" + } + ] + } + } + } + } + }, + "tags": [ + "Models" + ], + "parameters": [ + { + "name": "model_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ] + }, + "delete": { + "responses": { + "200": { + "description": "OK" + } + }, + "tags": [ + "Models" + ], + "parameters": [ + { + "name": "model_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ] + } + }, + "/v1/scoring-functions/{scoring_fn_id}": { "get": { "responses": { "200": { @@ -1185,7 +1227,7 @@ "parameters": [ { "name": "scoring_fn_id", - "in": "query", + "in": "path", "required": true, "schema": { "type": "string" @@ -1212,7 +1254,7 @@ ] } }, - "/alpha/shields/get": { + "/v1/shields/{identifier}": { "get": { "responses": { "200": { @@ -1239,7 +1281,7 @@ "parameters": [ { "name": "identifier", - "in": "query", + "in": "path", "required": true, "schema": { "type": "string" @@ -1266,18 +1308,70 @@ ] } }, - "/alpha/telemetry/get-span-tree": { - "post": { + "/v1/telemetry/traces/{trace_id}/spans/{span_id}": { + "get": { "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { - "type": "object", - "additionalProperties": { - "$ref": "#/components/schemas/SpanWithStatus" - } + "$ref": "#/components/schemas/Span" + } + } + } + } + }, + "tags": [ + "Telemetry" + ], + "parameters": [ + { + "name": "trace_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "span_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ] + } + }, + "/v1/telemetry/spans/{span_id}/tree": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/QuerySpanTreeResponse" } } } @@ -1289,12 +1383,23 @@ "parameters": [ { "name": "span_id", - "in": "query", + "in": "path", "required": true, "schema": { "type": "string" } }, + { + "name": "attributes_to_return", + "in": "query", + "required": false, + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, { "name": "max_depth", "in": "query", @@ -1321,20 +1426,10 @@ "type": "string" } } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/GetSpanTreeRequest" - } - } - }, - "required": true - } + ] } }, - "/alpha/tools/get": { + "/v1/tools/{tool_name}": { "get": { "responses": { "200": { @@ -1354,7 +1449,7 @@ "parameters": [ { "name": "tool_name", - "in": "query", + "in": "path", "required": true, "schema": { "type": "string" @@ -1381,7 +1476,7 @@ ] } }, - "/alpha/toolgroups/get": { + "/v1/toolgroups/{toolgroup_id}": { "get": { "responses": { "200": { @@ -1401,7 +1496,46 @@ "parameters": [ { "name": "toolgroup_id", - "in": "query", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ] + }, + "delete": { + "responses": { + "200": { + "description": "OK" + } + }, + "tags": [ + "ToolGroups" + ], + "summary": "Unregister a tool group", + "parameters": [ + { + "name": "toolgroup_id", + "in": "path", "required": true, "schema": { "type": "string" @@ -1428,7 +1562,54 @@ ] } }, - "/alpha/post-training/job/artifacts": { + "/v1/telemetry/traces/{trace_id}": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Trace" + } + } + } + } + }, + "tags": [ + "Telemetry" + ], + "parameters": [ + { + "name": "trace_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ] + } + }, + "/v1/post-training/job/artifacts": { "get": { "responses": { "200": { @@ -1482,7 +1663,7 @@ ] } }, - "/alpha/post-training/job/status": { + "/v1/post-training/job/status": { "get": { "responses": { "200": { @@ -1536,15 +1717,15 @@ ] } }, - "/alpha/post-training/jobs": { + "/v1/post-training/jobs": { "get": { "responses": { "200": { "description": "OK", "content": { - "application/jsonl": { + "application/json": { "schema": { - "$ref": "#/components/schemas/PostTrainingJob" + "$ref": "#/components/schemas/ListPostTrainingJobsResponse" } } } @@ -1575,7 +1756,99 @@ ] } }, - "/alpha/health": { + "/v1/vector-dbs/{vector_db_id}": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/VectorDB" + }, + { + "type": "null" + } + ] + } + } + } + } + }, + "tags": [ + "VectorDBs" + ], + "parameters": [ + { + "name": "vector_db_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ] + }, + "delete": { + "responses": { + "200": { + "description": "OK" + } + }, + "tags": [ + "VectorDBs" + ], + "parameters": [ + { + "name": "vector_db_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ] + } + }, + "/v1/health": { "get": { "responses": { "200": { @@ -1614,7 +1887,7 @@ ] } }, - "/alpha/memory/insert": { + "/v1/tool-runtime/rag-tool/insert": { "post": { "responses": { "200": { @@ -1622,7 +1895,50 @@ } }, "tags": [ - "Memory" + "ToolRuntime" + ], + "summary": "Index documents so they can be used by the RAG system", + "parameters": [ + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/InsertRequest" + } + } + }, + "required": true + } + } + }, + "/v1/vector-io/insert": { + "post": { + "responses": { + "200": { + "description": "OK" + } + }, + "tags": [ + "VectorIO" ], "parameters": [ { @@ -1648,7 +1964,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/InsertDocumentsRequest" + "$ref": "#/components/schemas/InsertChunksRequest" } } }, @@ -1656,7 +1972,7 @@ } } }, - "/alpha/tool-runtime/invoke": { + "/v1/tool-runtime/invoke": { "post": { "responses": { "200": { @@ -1706,104 +2022,7 @@ } } }, - "/alpha/eval/job/cancel": { - "post": { - "responses": { - "200": { - "description": "OK" - } - }, - "tags": [ - "Eval" - ], - "parameters": [ - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/JobCancelRequest" - } - } - }, - "required": true - } - } - }, - "/alpha/eval/job/result": { - "get": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/EvaluateResponse" - } - } - } - } - }, - "tags": [ - "Eval" - ], - "parameters": [ - { - "name": "task_id", - "in": "query", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "job_id", - "in": "query", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ] - } - }, - "/alpha/eval/job/status": { + "/v1/eval/tasks/{task_id}/jobs/{job_id}": { "get": { "responses": { "200": { @@ -1830,7 +2049,7 @@ "parameters": [ { "name": "task_id", - "in": "query", + "in": "path", "required": true, "schema": { "type": "string" @@ -1838,7 +2057,53 @@ }, { "name": "job_id", - "in": "query", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ] + }, + "delete": { + "responses": { + "200": { + "description": "OK" + } + }, + "tags": [ + "Eval" + ], + "parameters": [ + { + "name": "task_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "job_id", + "in": "path", "required": true, "schema": { "type": "string" @@ -1865,15 +2130,70 @@ ] } }, - "/alpha/datasets/list": { + "/v1/eval/tasks/{task_id}/jobs/{job_id}/result": { "get": { "responses": { "200": { "description": "OK", "content": { - "application/jsonl": { + "application/json": { "schema": { - "$ref": "#/components/schemas/Dataset" + "$ref": "#/components/schemas/EvaluateResponse" + } + } + } + } + }, + "tags": [ + "Eval" + ], + "parameters": [ + { + "name": "job_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "task_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ] + } + }, + "/v1/datasets": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListDatasetsResponse" } } } @@ -1902,687 +2222,7 @@ } } ] - } - }, - "/alpha/eval-tasks/list": { - "get": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/jsonl": { - "schema": { - "$ref": "#/components/schemas/EvalTask" - } - } - } - } - }, - "tags": [ - "EvalTasks" - ], - "parameters": [ - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ] - } - }, - "/alpha/memory-banks/list": { - "get": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/jsonl": { - "schema": { - "oneOf": [ - { - "$ref": "#/components/schemas/VectorMemoryBank" - }, - { - "$ref": "#/components/schemas/KeyValueMemoryBank" - }, - { - "$ref": "#/components/schemas/KeywordMemoryBank" - }, - { - "$ref": "#/components/schemas/GraphMemoryBank" - } - ] - } - } - } - } - }, - "tags": [ - "MemoryBanks" - ], - "parameters": [ - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ] - } - }, - "/alpha/models/list": { - "get": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/jsonl": { - "schema": { - "$ref": "#/components/schemas/Model" - } - } - } - } - }, - "tags": [ - "Models" - ], - "parameters": [ - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ] - } - }, - "/alpha/providers/list": { - "get": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": { - "$ref": "#/components/schemas/ProviderInfo" - } - } - } - } - } - }, - "tags": [ - "Inspect" - ], - "parameters": [ - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ] - } - }, - "/alpha/routes/list": { - "get": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/json": { - "schema": { - "type": "object", - "additionalProperties": { - "type": "array", - "items": { - "$ref": "#/components/schemas/RouteInfo" - } - } - } - } - } - } - }, - "tags": [ - "Inspect" - ], - "parameters": [ - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ] - } - }, - "/alpha/tool-runtime/list-tools": { - "post": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/jsonl": { - "schema": { - "$ref": "#/components/schemas/ToolDef" - } - } - } - } - }, - "tags": [ - "ToolRuntime" - ], - "parameters": [ - { - "name": "tool_group_id", - "in": "query", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ListRuntimeToolsRequest" - } - } - }, - "required": true - } - } - }, - "/alpha/scoring-functions/list": { - "get": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/jsonl": { - "schema": { - "$ref": "#/components/schemas/ScoringFn" - } - } - } - } - }, - "tags": [ - "ScoringFunctions" - ], - "parameters": [ - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ] - } - }, - "/alpha/shields/list": { - "get": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/jsonl": { - "schema": { - "$ref": "#/components/schemas/Shield" - } - } - } - } - }, - "tags": [ - "Shields" - ], - "parameters": [ - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ] - } - }, - "/alpha/toolgroups/list": { - "get": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/jsonl": { - "schema": { - "$ref": "#/components/schemas/ToolGroup" - } - } - } - } - }, - "tags": [ - "ToolGroups" - ], - "summary": "List tool groups with optional provider", - "parameters": [ - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ] - } - }, - "/alpha/tools/list": { - "get": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/jsonl": { - "schema": { - "$ref": "#/components/schemas/Tool" - } - } - } - } - }, - "tags": [ - "ToolGroups" - ], - "summary": "List tools with optional tool group", - "parameters": [ - { - "name": "tool_group_id", - "in": "query", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ] - } - }, - "/alpha/telemetry/log-event": { - "post": { - "responses": { - "200": { - "description": "OK" - } - }, - "tags": [ - "Telemetry" - ], - "parameters": [ - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/LogEventRequest" - } - } - }, - "required": true - } - } - }, - "/alpha/post-training/preference-optimize": { - "post": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/PostTrainingJob" - } - } - } - } - }, - "tags": [ - "PostTraining (Coming Soon)" - ], - "parameters": [ - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/PreferenceOptimizeRequest" - } - } - }, - "required": true - } - } - }, - "/alpha/memory/query": { - "post": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/QueryDocumentsResponse" - } - } - } - } - }, - "tags": [ - "Memory" - ], - "parameters": [ - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/QueryDocumentsRequest" - } - } - }, - "required": true - } - } - }, - "/alpha/telemetry/query-spans": { - "post": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/jsonl": { - "schema": { - "$ref": "#/components/schemas/Span" - } - } - } - } - }, - "tags": [ - "Telemetry" - ], - "parameters": [ - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/QuerySpansRequest" - } - } - }, - "required": true - } - } - }, - "/alpha/telemetry/query-traces": { - "post": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/jsonl": { - "schema": { - "$ref": "#/components/schemas/Trace" - } - } - } - } - }, - "tags": [ - "Telemetry" - ], - "parameters": [ - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/QueryTracesRequest" - } - } - }, - "required": true - } - } - }, - "/alpha/datasets/register": { + }, "post": { "responses": { "200": { @@ -2624,7 +2264,44 @@ } } }, - "/alpha/eval-tasks/register": { + "/v1/eval-tasks": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListEvalTasksResponse" + } + } + } + } + }, + "tags": [ + "EvalTasks" + ], + "parameters": [ + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ] + }, "post": { "responses": { "200": { @@ -2666,11 +2343,22 @@ } } }, - "/alpha/memory-banks/register": { - "post": { - "responses": {}, + "/v1/models": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListModelsResponse" + } + } + } + } + }, "tags": [ - "MemoryBanks" + "Models" ], "parameters": [ { @@ -2691,20 +2379,8 @@ "type": "string" } } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/RegisterMemoryBankRequest" - } - } - }, - "required": true - } - } - }, - "/alpha/models/register": { + ] + }, "post": { "responses": { "200": { @@ -2753,7 +2429,177 @@ } } }, - "/alpha/scoring-functions/register": { + "/v1/inspect/providers": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListProvidersResponse" + } + } + } + } + }, + "tags": [ + "Inspect" + ], + "parameters": [ + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ] + } + }, + "/v1/inspect/routes": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListRoutesResponse" + } + } + } + } + }, + "tags": [ + "Inspect" + ], + "parameters": [ + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ] + } + }, + "/v1/tool-runtime/list-tools": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/jsonl": { + "schema": { + "$ref": "#/components/schemas/ToolDef" + } + } + } + } + }, + "tags": [ + "ToolRuntime" + ], + "parameters": [ + { + "name": "tool_group_id", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "mcp_endpoint", + "in": "query", + "required": false, + "schema": { + "$ref": "#/components/schemas/URL" + } + }, + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ] + } + }, + "/v1/scoring-functions": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListScoringFunctionsResponse" + } + } + } + } + }, + "tags": [ + "ScoringFunctions" + ], + "parameters": [ + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ] + }, "post": { "responses": { "200": { @@ -2795,7 +2641,44 @@ } } }, - "/alpha/shields/register": { + "/v1/shields": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListShieldsResponse" + } + } + } + } + }, + "tags": [ + "Shields" + ], + "parameters": [ + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ] + }, "post": { "responses": { "200": { @@ -2844,7 +2727,45 @@ } } }, - "/alpha/toolgroups/register": { + "/v1/toolgroups": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListToolGroupsResponse" + } + } + } + } + }, + "tags": [ + "ToolGroups" + ], + "summary": "List tool groups with optional provider", + "parameters": [ + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ] + }, "post": { "responses": { "200": { @@ -2887,7 +2808,477 @@ } } }, - "/alpha/eval/run-eval": { + "/v1/tools": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListToolsResponse" + } + } + } + } + }, + "tags": [ + "ToolGroups" + ], + "summary": "List tools with optional tool group", + "parameters": [ + { + "name": "toolgroup_id", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ] + } + }, + "/v1/vector-dbs": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListVectorDBsResponse" + } + } + } + } + }, + "tags": [ + "VectorDBs" + ], + "parameters": [ + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ] + }, + "post": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VectorDB" + } + } + } + } + }, + "tags": [ + "VectorDBs" + ], + "parameters": [ + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RegisterVectorDbRequest" + } + } + }, + "required": true + } + } + }, + "/v1/telemetry/events": { + "post": { + "responses": { + "200": { + "description": "OK" + } + }, + "tags": [ + "Telemetry" + ], + "parameters": [ + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/LogEventRequest" + } + } + }, + "required": true + } + } + }, + "/v1/post-training/preference-optimize": { + "post": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PostTrainingJob" + } + } + } + } + }, + "tags": [ + "PostTraining (Coming Soon)" + ], + "parameters": [ + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PreferenceOptimizeRequest" + } + } + }, + "required": true + } + } + }, + "/v1/tool-runtime/rag-tool/query": { + "post": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RAGQueryResult" + } + } + } + } + }, + "tags": [ + "ToolRuntime" + ], + "summary": "Query the RAG system for context; typically invoked by the agent", + "parameters": [ + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/QueryRequest" + } + } + }, + "required": true + } + } + }, + "/v1/vector-io/query": { + "post": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/QueryChunksResponse" + } + } + } + } + }, + "tags": [ + "VectorIO" + ], + "parameters": [ + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/QueryChunksRequest" + } + } + }, + "required": true + } + } + }, + "/v1/telemetry/spans": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/QuerySpansResponse" + } + } + } + } + }, + "tags": [ + "Telemetry" + ], + "parameters": [ + { + "name": "attribute_filters", + "in": "query", + "required": true, + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/QueryCondition" + } + } + }, + { + "name": "attributes_to_return", + "in": "query", + "required": true, + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + { + "name": "max_depth", + "in": "query", + "required": false, + "schema": { + "type": "integer" + } + }, + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ] + } + }, + "/v1/telemetry/traces": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/QueryTracesResponse" + } + } + } + } + }, + "tags": [ + "Telemetry" + ], + "parameters": [ + { + "name": "attribute_filters", + "in": "query", + "required": false, + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/QueryCondition" + } + } + }, + { + "name": "limit", + "in": "query", + "required": false, + "schema": { + "type": "integer" + } + }, + { + "name": "offset", + "in": "query", + "required": false, + "schema": { + "type": "integer" + } + }, + { + "name": "order_by", + "in": "query", + "required": false, + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + }, + { + "name": "X-LlamaStack-Provider-Data", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-Client-Version", + "in": "header", + "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", + "required": false, + "schema": { + "type": "string" + } + } + ] + } + }, + "/v1/eval/tasks/{task_id}/jobs": { "post": { "responses": { "200": { @@ -2905,6 +3296,14 @@ "Eval" ], "parameters": [ + { + "name": "task_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, { "name": "X-LlamaStack-Provider-Data", "in": "header", @@ -2936,7 +3335,7 @@ } } }, - "/alpha/safety/run-shield": { + "/v1/safety/run-shield": { "post": { "responses": { "200": { @@ -2985,7 +3384,7 @@ } } }, - "/alpha/telemetry/save-spans-to-dataset": { + "/v1/telemetry/spans/export": { "post": { "responses": { "200": { @@ -3027,7 +3426,7 @@ } } }, - "/alpha/scoring/score": { + "/v1/scoring/score": { "post": { "responses": { "200": { @@ -3076,7 +3475,7 @@ } } }, - "/alpha/scoring/score-batch": { + "/v1/scoring/score-batch": { "post": { "responses": { "200": { @@ -3125,7 +3524,7 @@ } } }, - "/alpha/post-training/supervised-fine-tune": { + "/v1/post-training/supervised-fine-tune": { "post": { "responses": { "200": { @@ -3174,7 +3573,7 @@ } } }, - "/alpha/synthetic-data-generation/generate": { + "/v1/synthetic-data-generation/generate": { "post": { "responses": { "200": { @@ -3223,176 +3622,7 @@ } } }, - "/alpha/datasets/unregister": { - "post": { - "responses": { - "200": { - "description": "OK" - } - }, - "tags": [ - "Datasets" - ], - "parameters": [ - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/UnregisterDatasetRequest" - } - } - }, - "required": true - } - } - }, - "/alpha/memory-banks/unregister": { - "post": { - "responses": { - "200": { - "description": "OK" - } - }, - "tags": [ - "MemoryBanks" - ], - "parameters": [ - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/UnregisterMemoryBankRequest" - } - } - }, - "required": true - } - } - }, - "/alpha/models/unregister": { - "post": { - "responses": { - "200": { - "description": "OK" - } - }, - "tags": [ - "Models" - ], - "parameters": [ - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/UnregisterModelRequest" - } - } - }, - "required": true - } - } - }, - "/alpha/toolgroups/unregister": { - "post": { - "responses": { - "200": { - "description": "OK" - } - }, - "tags": [ - "ToolGroups" - ], - "summary": "Unregister a tool group", - "parameters": [ - { - "name": "X-LlamaStack-Provider-Data", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-Client-Version", - "in": "header", - "description": "Version of the client making the request. This is used to ensure that the client and server are compatible.", - "required": false, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/UnregisterToolGroupRequest" - } - } - }, - "required": true - } - } - }, - "/alpha/version": { + "/v1/version": { "get": { "responses": { "200": { @@ -3514,20 +3744,13 @@ "tool_calls" ] }, - "ImageContentItem": { + "GreedySamplingStrategy": { "type": "object", "properties": { - "url": { - "$ref": "#/components/schemas/URL" - }, - "data": { - "type": "string", - "contentEncoding": "base64" - }, "type": { "type": "string", - "const": "image", - "default": "image" + "const": "greedy", + "default": "greedy" } }, "additionalProperties": false, @@ -3535,6 +3758,34 @@ "type" ] }, + "ImageContentItem": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "image", + "default": "image" + }, + "image": { + "type": "object", + "properties": { + "url": { + "$ref": "#/components/schemas/URL" + }, + "data": { + "type": "string", + "contentEncoding": "base64" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false, + "required": [ + "type", + "image" + ] + }, "InterleavedContent": { "oneOf": [ { @@ -3559,7 +3810,10 @@ { "$ref": "#/components/schemas/TextContentItem" } - ] + ], + "discriminator": { + "propertyName": "type" + } }, "Message": { "oneOf": [ @@ -3575,26 +3829,29 @@ { "$ref": "#/components/schemas/CompletionMessage" } - ] + ], + "discriminator": { + "propertyName": "role" + } }, "SamplingParams": { "type": "object", "properties": { "strategy": { - "$ref": "#/components/schemas/SamplingStrategy", - "default": "greedy" - }, - "temperature": { - "type": "number", - "default": 0.0 - }, - "top_p": { - "type": "number", - "default": 0.95 - }, - "top_k": { - "type": "integer", - "default": 0 + "oneOf": [ + { + "$ref": "#/components/schemas/GreedySamplingStrategy" + }, + { + "$ref": "#/components/schemas/TopPSamplingStrategy" + }, + { + "$ref": "#/components/schemas/TopKSamplingStrategy" + } + ], + "discriminator": { + "propertyName": "type" + } }, "max_tokens": { "type": "integer", @@ -3610,14 +3867,6 @@ "strategy" ] }, - "SamplingStrategy": { - "type": "string", - "enum": [ - "greedy", - "top_p", - "top_k" - ] - }, "StopReason": { "type": "string", "enum": [ @@ -3871,6 +4120,45 @@ "content" ] }, + "TopKSamplingStrategy": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "top_k", + "default": "top_k" + }, + "top_k": { + "type": "integer" + } + }, + "additionalProperties": false, + "required": [ + "type", + "top_k" + ] + }, + "TopPSamplingStrategy": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "top_p", + "default": "top_p" + }, + "temperature": { + "type": "number" + }, + "top_p": { + "type": "number", + "default": 0.95 + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, "URL": { "type": "object", "properties": { @@ -4107,7 +4395,10 @@ "bnf" ] } - ] + ], + "discriminator": { + "propertyName": "type" + } }, "ChatCompletionRequest": { "type": "object", @@ -4228,45 +4519,54 @@ "ContentDelta": { "oneOf": [ { - "type": "object", - "properties": { - "type": { - "type": "string", - "const": "text", - "default": "text" - }, - "text": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "type", - "text" - ] + "$ref": "#/components/schemas/TextDelta" }, { - "type": "object", - "properties": { - "type": { - "type": "string", - "const": "image", - "default": "image" - }, - "data": { - "type": "string", - "contentEncoding": "base64" - } - }, - "additionalProperties": false, - "required": [ - "type", - "data" - ] + "$ref": "#/components/schemas/ImageDelta" }, { "$ref": "#/components/schemas/ToolCallDelta" } + ], + "discriminator": { + "propertyName": "type" + } + }, + "ImageDelta": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "image", + "default": "image" + }, + "image": { + "type": "string", + "contentEncoding": "base64" + } + }, + "additionalProperties": false, + "required": [ + "type", + "image" + ] + }, + "TextDelta": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "text", + "default": "text" + }, + "text": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "type", + "text" ] }, "TokenLogProbs": { @@ -4292,7 +4592,7 @@ "const": "tool_call", "default": "tool_call" }, - "content": { + "tool_call": { "oneOf": [ { "type": "string" @@ -4309,7 +4609,7 @@ "additionalProperties": false, "required": [ "type", - "content", + "tool_call", "parse_status" ] }, @@ -4627,16 +4927,12 @@ "CreateAgentSessionRequest": { "type": "object", "properties": { - "agent_id": { - "type": "string" - }, "session_name": { "type": "string" } }, "additionalProperties": false, "required": [ - "agent_id", "session_name" ] }, @@ -4655,12 +4951,6 @@ "CreateAgentTurnRequest": { "type": "object", "properties": { - "agent_id": { - "type": "string" - }, - "session_id": { - "type": "string" - }, "messages": { "type": "array", "items": { @@ -4721,8 +5011,6 @@ }, "additionalProperties": false, "required": [ - "agent_id", - "session_id", "messages" ] }, @@ -4746,7 +5034,10 @@ { "$ref": "#/components/schemas/AgentTurnResponseTurnCompletePayload" } - ] + ], + "discriminator": { + "propertyName": "event_type" + } } }, "additionalProperties": false, @@ -4789,7 +5080,10 @@ { "$ref": "#/components/schemas/MemoryRetrievalStep" } - ] + ], + "discriminator": { + "propertyName": "step_type" + } } }, "additionalProperties": false, @@ -4990,11 +5284,8 @@ "const": "memory_retrieval", "default": "memory_retrieval" }, - "memory_bank_ids": { - "type": "array", - "items": { - "type": "string" - } + "vector_db_ids": { + "type": "string" }, "inserted_context": { "$ref": "#/components/schemas/InterleavedContent" @@ -5005,7 +5296,7 @@ "turn_id", "step_id", "step_type", - "memory_bank_ids", + "vector_db_ids", "inserted_context" ] }, @@ -5192,7 +5483,10 @@ { "$ref": "#/components/schemas/MemoryRetrievalStep" } - ] + ], + "discriminator": { + "propertyName": "step_type" + } } }, "output_message": { @@ -5262,34 +5556,6 @@ "error" ] }, - "DeleteAgentsRequest": { - "type": "object", - "properties": { - "agent_id": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "agent_id" - ] - }, - "DeleteAgentsSessionRequest": { - "type": "object", - "properties": { - "agent_id": { - "type": "string" - }, - "session_id": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "agent_id", - "session_id" - ] - }, "EmbeddingsRequest": { "type": "object", "properties": { @@ -5370,7 +5636,10 @@ { "$ref": "#/components/schemas/AgentCandidate" } - ] + ], + "discriminator": { + "propertyName": "type" + } }, "scoring_params": { "type": "object", @@ -5385,7 +5654,10 @@ { "$ref": "#/components/schemas/BasicScoringFnParams" } - ] + ], + "discriminator": { + "propertyName": "type" + } } }, "num_examples": { @@ -5435,7 +5707,10 @@ { "$ref": "#/components/schemas/AgentCandidate" } - ] + ], + "discriminator": { + "propertyName": "type" + } }, "num_examples": { "type": "integer" @@ -5534,9 +5809,6 @@ "EvaluateRowsRequest": { "type": "object", "properties": { - "task_id": { - "type": "string" - }, "input_rows": { "type": "array", "items": { @@ -5579,12 +5851,14 @@ { "$ref": "#/components/schemas/AppEvalTaskConfig" } - ] + ], + "discriminator": { + "propertyName": "type" + } } }, "additionalProperties": false, "required": [ - "task_id", "input_rows", "scoring_functions", "task_config" @@ -5697,114 +5971,6 @@ "aggregated_results" ] }, - "GetAgentsSessionRequest": { - "type": "object", - "properties": { - "turn_ids": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "additionalProperties": false - }, - "GraphMemoryBank": { - "type": "object", - "properties": { - "identifier": { - "type": "string" - }, - "provider_resource_id": { - "type": "string" - }, - "provider_id": { - "type": "string" - }, - "type": { - "type": "string", - "const": "memory_bank", - "default": "memory_bank" - }, - "memory_bank_type": { - "type": "string", - "const": "graph", - "default": "graph" - } - }, - "additionalProperties": false, - "required": [ - "identifier", - "provider_resource_id", - "provider_id", - "type", - "memory_bank_type" - ] - }, - "KeyValueMemoryBank": { - "type": "object", - "properties": { - "identifier": { - "type": "string" - }, - "provider_resource_id": { - "type": "string" - }, - "provider_id": { - "type": "string" - }, - "type": { - "type": "string", - "const": "memory_bank", - "default": "memory_bank" - }, - "memory_bank_type": { - "type": "string", - "const": "keyvalue", - "default": "keyvalue" - } - }, - "additionalProperties": false, - "required": [ - "identifier", - "provider_resource_id", - "provider_id", - "type", - "memory_bank_type" - ] - }, - "KeywordMemoryBank": { - "type": "object", - "properties": { - "identifier": { - "type": "string" - }, - "provider_resource_id": { - "type": "string" - }, - "provider_id": { - "type": "string" - }, - "type": { - "type": "string", - "const": "memory_bank", - "default": "memory_bank" - }, - "memory_bank_type": { - "type": "string", - "const": "keyword", - "default": "keyword" - } - }, - "additionalProperties": false, - "required": [ - "identifier", - "provider_resource_id", - "provider_id", - "type", - "memory_bank_type" - ] - }, "Session": { "type": "object", "properties": { @@ -5823,22 +5989,6 @@ "started_at": { "type": "string", "format": "date-time" - }, - "memory_bank": { - "oneOf": [ - { - "$ref": "#/components/schemas/VectorMemoryBank" - }, - { - "$ref": "#/components/schemas/KeyValueMemoryBank" - }, - { - "$ref": "#/components/schemas/KeywordMemoryBank" - }, - { - "$ref": "#/components/schemas/GraphMemoryBank" - } - ] } }, "additionalProperties": false, @@ -5850,53 +6000,6 @@ ], "title": "A single session of an interaction with an Agentic System." }, - "VectorMemoryBank": { - "type": "object", - "properties": { - "identifier": { - "type": "string" - }, - "provider_resource_id": { - "type": "string" - }, - "provider_id": { - "type": "string" - }, - "type": { - "type": "string", - "const": "memory_bank", - "default": "memory_bank" - }, - "memory_bank_type": { - "type": "string", - "const": "vector", - "default": "vector" - }, - "embedding_model": { - "type": "string" - }, - "chunk_size_in_tokens": { - "type": "integer" - }, - "embedding_dimension": { - "type": "integer", - "default": 384 - }, - "overlap_size_in_tokens": { - "type": "integer" - } - }, - "additionalProperties": false, - "required": [ - "identifier", - "provider_resource_id", - "provider_id", - "type", - "memory_bank_type", - "embedding_model", - "chunk_size_in_tokens" - ] - }, "AgentStepResponse": { "type": "object", "properties": { @@ -5914,7 +6017,10 @@ { "$ref": "#/components/schemas/MemoryRetrievalStep" } - ] + ], + "discriminator": { + "propertyName": "step_type" + } } }, "additionalProperties": false, @@ -5922,6 +6028,76 @@ "step" ] }, + "AgentTurnInputType": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "agent_turn_input", + "default": "agent_turn_input" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + "ArrayType": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "array", + "default": "array" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + "BooleanType": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "boolean", + "default": "boolean" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + "ChatCompletionInputType": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "chat_completion_input", + "default": "chat_completion_input" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + "CompletionInputType": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "completion_input", + "default": "completion_input" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, "Dataset": { "type": "object", "properties": { @@ -5985,148 +6161,111 @@ "metadata" ] }, + "JsonType": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "json", + "default": "json" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + "NumberType": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "number", + "default": "number" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + "ObjectType": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "object", + "default": "object" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, "ParamType": { "oneOf": [ { - "type": "object", - "properties": { - "type": { - "type": "string", - "const": "string", - "default": "string" - } - }, - "additionalProperties": false, - "required": [ - "type" - ] + "$ref": "#/components/schemas/StringType" }, { - "type": "object", - "properties": { - "type": { - "type": "string", - "const": "number", - "default": "number" - } - }, - "additionalProperties": false, - "required": [ - "type" - ] + "$ref": "#/components/schemas/NumberType" }, { - "type": "object", - "properties": { - "type": { - "type": "string", - "const": "boolean", - "default": "boolean" - } - }, - "additionalProperties": false, - "required": [ - "type" - ] + "$ref": "#/components/schemas/BooleanType" }, { - "type": "object", - "properties": { - "type": { - "type": "string", - "const": "array", - "default": "array" - } - }, - "additionalProperties": false, - "required": [ - "type" - ] + "$ref": "#/components/schemas/ArrayType" }, { - "type": "object", - "properties": { - "type": { - "type": "string", - "const": "object", - "default": "object" - } - }, - "additionalProperties": false, - "required": [ - "type" - ] + "$ref": "#/components/schemas/ObjectType" }, { - "type": "object", - "properties": { - "type": { - "type": "string", - "const": "json", - "default": "json" - } - }, - "additionalProperties": false, - "required": [ - "type" - ] + "$ref": "#/components/schemas/JsonType" }, { - "type": "object", - "properties": { - "type": { - "type": "string", - "const": "union", - "default": "union" - } - }, - "additionalProperties": false, - "required": [ - "type" - ] + "$ref": "#/components/schemas/UnionType" }, { - "type": "object", - "properties": { - "type": { - "type": "string", - "const": "chat_completion_input", - "default": "chat_completion_input" - } - }, - "additionalProperties": false, - "required": [ - "type" - ] + "$ref": "#/components/schemas/ChatCompletionInputType" }, { - "type": "object", - "properties": { - "type": { - "type": "string", - "const": "completion_input", - "default": "completion_input" - } - }, - "additionalProperties": false, - "required": [ - "type" - ] + "$ref": "#/components/schemas/CompletionInputType" }, { - "type": "object", - "properties": { - "type": { - "type": "string", - "const": "agent_turn_input", - "default": "agent_turn_input" - } - }, - "additionalProperties": false, - "required": [ - "type" - ] + "$ref": "#/components/schemas/AgentTurnInputType" } + ], + "discriminator": { + "propertyName": "type" + } + }, + "StringType": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "string", + "default": "string" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + "UnionType": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "union", + "default": "union" + } + }, + "additionalProperties": false, + "required": [ + "type" ] }, "EvalTask": { @@ -6359,7 +6498,10 @@ { "$ref": "#/components/schemas/BasicScoringFnParams" } - ] + ], + "discriminator": { + "propertyName": "type" + } } }, "additionalProperties": false, @@ -6424,17 +6566,62 @@ ], "title": "A safety shield resource that can be used to check content" }, - "GetSpanTreeRequest": { + "Span": { "type": "object", "properties": { - "attributes_to_return": { - "type": "array", - "items": { - "type": "string" + "span_id": { + "type": "string" + }, + "trace_id": { + "type": "string" + }, + "parent_span_id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "start_time": { + "type": "string", + "format": "date-time" + }, + "end_time": { + "type": "string", + "format": "date-time" + }, + "attributes": { + "type": "object", + "additionalProperties": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "boolean" + }, + { + "type": "number" + }, + { + "type": "string" + }, + { + "type": "array" + }, + { + "type": "object" + } + ] } } }, - "additionalProperties": false + "additionalProperties": false, + "required": [ + "span_id", + "trace_id", + "name", + "start_time" + ] }, "SpanStatus": { "type": "string", @@ -6503,6 +6690,21 @@ "start_time" ] }, + "QuerySpanTreeResponse": { + "type": "object", + "properties": { + "data": { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/SpanWithStatus" + } + } + }, + "additionalProperties": false, + "required": [ + "data" + ] + }, "Tool": { "type": "object", "properties": { @@ -6635,6 +6837,31 @@ "type" ] }, + "Trace": { + "type": "object", + "properties": { + "trace_id": { + "type": "string" + }, + "root_span_id": { + "type": "string" + }, + "start_time": { + "type": "string", + "format": "date-time" + }, + "end_time": { + "type": "string", + "format": "date-time" + } + }, + "additionalProperties": false, + "required": [ + "trace_id", + "root_span_id", + "start_time" + ] + }, "Checkpoint": { "description": "Checkpoint created during training runs" }, @@ -6728,16 +6955,62 @@ ], "title": "Status of a finetuning job." }, - "PostTrainingJob": { + "ListPostTrainingJobsResponse": { "type": "object", "properties": { - "job_uuid": { - "type": "string" + "data": { + "type": "array", + "items": { + "type": "object", + "properties": { + "job_uuid": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "job_uuid" + ] + } } }, "additionalProperties": false, "required": [ - "job_uuid" + "data" + ] + }, + "VectorDB": { + "type": "object", + "properties": { + "identifier": { + "type": "string" + }, + "provider_resource_id": { + "type": "string" + }, + "provider_id": { + "type": "string" + }, + "type": { + "type": "string", + "const": "vector_db", + "default": "vector_db" + }, + "embedding_model": { + "type": "string" + }, + "embedding_dimension": { + "type": "integer" + } + }, + "additionalProperties": false, + "required": [ + "identifier", + "provider_resource_id", + "provider_id", + "type", + "embedding_model", + "embedding_dimension" ] }, "HealthInfo": { @@ -6752,7 +7025,7 @@ "status" ] }, - "MemoryBankDocument": { + "RAGDocument": { "type": "object", "properties": { "document_id": { @@ -6813,16 +7086,74 @@ "metadata" ] }, - "InsertDocumentsRequest": { + "InsertRequest": { "type": "object", "properties": { - "bank_id": { - "type": "string" - }, "documents": { "type": "array", "items": { - "$ref": "#/components/schemas/MemoryBankDocument" + "$ref": "#/components/schemas/RAGDocument" + } + }, + "vector_db_id": { + "type": "string" + }, + "chunk_size_in_tokens": { + "type": "integer" + } + }, + "additionalProperties": false, + "required": [ + "documents", + "vector_db_id", + "chunk_size_in_tokens" + ] + }, + "InsertChunksRequest": { + "type": "object", + "properties": { + "vector_db_id": { + "type": "string" + }, + "chunks": { + "type": "array", + "items": { + "type": "object", + "properties": { + "content": { + "$ref": "#/components/schemas/InterleavedContent" + }, + "metadata": { + "type": "object", + "additionalProperties": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "boolean" + }, + { + "type": "number" + }, + { + "type": "string" + }, + { + "type": "array" + }, + { + "type": "object" + } + ] + } + } + }, + "additionalProperties": false, + "required": [ + "content", + "metadata" + ] } }, "ttl_seconds": { @@ -6831,8 +7162,8 @@ }, "additionalProperties": false, "required": [ - "bank_id", - "documents" + "vector_db_id", + "chunks" ] }, "InvokeToolRequest": { @@ -6841,7 +7172,7 @@ "tool_name": { "type": "string" }, - "args": { + "kwargs": { "type": "object", "additionalProperties": { "oneOf": [ @@ -6870,7 +7201,7 @@ "additionalProperties": false, "required": [ "tool_name", - "args" + "kwargs" ] }, "ToolInvocationResult": { @@ -6891,25 +7222,57 @@ "content" ] }, - "JobCancelRequest": { + "ListDatasetsResponse": { "type": "object", "properties": { - "task_id": { - "type": "string" - }, - "job_id": { - "type": "string" + "data": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Dataset" + } } }, "additionalProperties": false, "required": [ - "task_id", - "job_id" + "data" + ] + }, + "ListEvalTasksResponse": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/components/schemas/EvalTask" + } + } + }, + "additionalProperties": false, + "required": [ + "data" + ] + }, + "ListModelsResponse": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Model" + } + } + }, + "additionalProperties": false, + "required": [ + "data" ] }, "ProviderInfo": { "type": "object", "properties": { + "api": { + "type": "string" + }, "provider_id": { "type": "string" }, @@ -6919,10 +7282,26 @@ }, "additionalProperties": false, "required": [ + "api", "provider_id", "provider_type" ] }, + "ListProvidersResponse": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ProviderInfo" + } + } + }, + "additionalProperties": false, + "required": [ + "data" + ] + }, "RouteInfo": { "type": "object", "properties": { @@ -6946,14 +7325,95 @@ "provider_types" ] }, - "ListRuntimeToolsRequest": { + "ListRoutesResponse": { "type": "object", "properties": { - "mcp_endpoint": { - "$ref": "#/components/schemas/URL" + "data": { + "type": "array", + "items": { + "$ref": "#/components/schemas/RouteInfo" + } } }, - "additionalProperties": false + "additionalProperties": false, + "required": [ + "data" + ] + }, + "ListScoringFunctionsResponse": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ScoringFn" + } + } + }, + "additionalProperties": false, + "required": [ + "data" + ] + }, + "ListShieldsResponse": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Shield" + } + } + }, + "additionalProperties": false, + "required": [ + "data" + ] + }, + "ListToolGroupsResponse": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ToolGroup" + } + } + }, + "additionalProperties": false, + "required": [ + "data" + ] + }, + "ListToolsResponse": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Tool" + } + } + }, + "additionalProperties": false, + "required": [ + "data" + ] + }, + "ListVectorDBsResponse": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/components/schemas/VectorDB" + } + } + }, + "additionalProperties": false, + "required": [ + "data" + ] }, "LogSeverity": { "type": "string", @@ -7127,7 +7587,10 @@ { "$ref": "#/components/schemas/SpanEndPayload" } - ] + ], + "discriminator": { + "propertyName": "type" + } } }, "additionalProperties": false, @@ -7213,7 +7676,10 @@ { "$ref": "#/components/schemas/StructuredLogEvent" } - ] + ], + "discriminator": { + "propertyName": "type" + } }, "ttl_seconds": { "type": "integer" @@ -7261,6 +7727,9 @@ "shuffle": { "type": "boolean" }, + "data_format": { + "$ref": "#/components/schemas/DatasetFormat" + }, "validation_dataset_id": { "type": "string" }, @@ -7277,7 +7746,15 @@ "required": [ "dataset_id", "batch_size", - "shuffle" + "shuffle", + "data_format" + ] + }, + "DatasetFormat": { + "type": "string", + "enum": [ + "instruct", + "dialog" ] }, "EfficiencyConfig": { @@ -7449,10 +7926,129 @@ "logger_config" ] }, - "QueryDocumentsRequest": { + "PostTrainingJob": { "type": "object", "properties": { - "bank_id": { + "job_uuid": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "job_uuid" + ] + }, + "DefaultRAGQueryGeneratorConfig": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "default", + "default": "default" + }, + "separator": { + "type": "string", + "default": " " + } + }, + "additionalProperties": false, + "required": [ + "type", + "separator" + ] + }, + "LLMRAGQueryGeneratorConfig": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "llm", + "default": "llm" + }, + "model": { + "type": "string" + }, + "template": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "type", + "model", + "template" + ] + }, + "RAGQueryConfig": { + "type": "object", + "properties": { + "query_generator_config": { + "$ref": "#/components/schemas/RAGQueryGeneratorConfig" + }, + "max_tokens_in_context": { + "type": "integer", + "default": 4096 + }, + "max_chunks": { + "type": "integer", + "default": 5 + } + }, + "additionalProperties": false, + "required": [ + "query_generator_config", + "max_tokens_in_context", + "max_chunks" + ] + }, + "RAGQueryGeneratorConfig": { + "oneOf": [ + { + "$ref": "#/components/schemas/DefaultRAGQueryGeneratorConfig" + }, + { + "$ref": "#/components/schemas/LLMRAGQueryGeneratorConfig" + } + ], + "discriminator": { + "propertyName": "type" + } + }, + "QueryRequest": { + "type": "object", + "properties": { + "content": { + "$ref": "#/components/schemas/InterleavedContent" + }, + "vector_db_ids": { + "type": "array", + "items": { + "type": "string" + } + }, + "query_config": { + "$ref": "#/components/schemas/RAGQueryConfig" + } + }, + "additionalProperties": false, + "required": [ + "content", + "vector_db_ids" + ] + }, + "RAGQueryResult": { + "type": "object", + "properties": { + "content": { + "$ref": "#/components/schemas/InterleavedContent" + } + }, + "additionalProperties": false + }, + "QueryChunksRequest": { + "type": "object", + "properties": { + "vector_db_id": { "type": "string" }, "query": { @@ -7486,11 +8082,11 @@ }, "additionalProperties": false, "required": [ - "bank_id", + "vector_db_id", "query" ] }, - "QueryDocumentsResponse": { + "QueryChunksResponse": { "type": "object", "properties": { "chunks": { @@ -7501,18 +8097,36 @@ "content": { "$ref": "#/components/schemas/InterleavedContent" }, - "token_count": { - "type": "integer" - }, - "document_id": { - "type": "string" + "metadata": { + "type": "object", + "additionalProperties": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "boolean" + }, + { + "type": "number" + }, + { + "type": "string" + }, + { + "type": "array" + }, + { + "type": "object" + } + ] + } } }, "additionalProperties": false, "required": [ "content", - "token_count", - "document_id" + "metadata" ] } }, @@ -7577,135 +8191,34 @@ "lt" ] }, - "QuerySpansRequest": { + "QuerySpansResponse": { "type": "object", "properties": { - "attribute_filters": { + "data": { "type": "array", "items": { - "$ref": "#/components/schemas/QueryCondition" + "$ref": "#/components/schemas/Span" } - }, - "attributes_to_return": { - "type": "array", - "items": { - "type": "string" - } - }, - "max_depth": { - "type": "integer" } }, "additionalProperties": false, "required": [ - "attribute_filters", - "attributes_to_return" + "data" ] }, - "Span": { + "QueryTracesResponse": { "type": "object", "properties": { - "span_id": { - "type": "string" - }, - "trace_id": { - "type": "string" - }, - "parent_span_id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "start_time": { - "type": "string", - "format": "date-time" - }, - "end_time": { - "type": "string", - "format": "date-time" - }, - "attributes": { - "type": "object", - "additionalProperties": { - "oneOf": [ - { - "type": "null" - }, - { - "type": "boolean" - }, - { - "type": "number" - }, - { - "type": "string" - }, - { - "type": "array" - }, - { - "type": "object" - } - ] + "data": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Trace" } } }, "additionalProperties": false, "required": [ - "span_id", - "trace_id", - "name", - "start_time" - ] - }, - "QueryTracesRequest": { - "type": "object", - "properties": { - "attribute_filters": { - "type": "array", - "items": { - "$ref": "#/components/schemas/QueryCondition" - } - }, - "limit": { - "type": "integer" - }, - "offset": { - "type": "integer" - }, - "order_by": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "additionalProperties": false - }, - "Trace": { - "type": "object", - "properties": { - "trace_id": { - "type": "string" - }, - "root_span_id": { - "type": "string" - }, - "start_time": { - "type": "string", - "format": "date-time" - }, - "end_time": { - "type": "string", - "format": "date-time" - } - }, - "additionalProperties": false, - "required": [ - "trace_id", - "root_span_id", - "start_time" + "data" ] }, "RegisterDatasetRequest": { @@ -7816,108 +8329,6 @@ "scoring_functions" ] }, - "GraphMemoryBankParams": { - "type": "object", - "properties": { - "memory_bank_type": { - "type": "string", - "const": "graph", - "default": "graph" - } - }, - "additionalProperties": false, - "required": [ - "memory_bank_type" - ] - }, - "KeyValueMemoryBankParams": { - "type": "object", - "properties": { - "memory_bank_type": { - "type": "string", - "const": "keyvalue", - "default": "keyvalue" - } - }, - "additionalProperties": false, - "required": [ - "memory_bank_type" - ] - }, - "KeywordMemoryBankParams": { - "type": "object", - "properties": { - "memory_bank_type": { - "type": "string", - "const": "keyword", - "default": "keyword" - } - }, - "additionalProperties": false, - "required": [ - "memory_bank_type" - ] - }, - "VectorMemoryBankParams": { - "type": "object", - "properties": { - "memory_bank_type": { - "type": "string", - "const": "vector", - "default": "vector" - }, - "embedding_model": { - "type": "string" - }, - "chunk_size_in_tokens": { - "type": "integer" - }, - "overlap_size_in_tokens": { - "type": "integer" - } - }, - "additionalProperties": false, - "required": [ - "memory_bank_type", - "embedding_model", - "chunk_size_in_tokens" - ] - }, - "RegisterMemoryBankRequest": { - "type": "object", - "properties": { - "memory_bank_id": { - "type": "string" - }, - "params": { - "oneOf": [ - { - "$ref": "#/components/schemas/VectorMemoryBankParams" - }, - { - "$ref": "#/components/schemas/KeyValueMemoryBankParams" - }, - { - "$ref": "#/components/schemas/KeywordMemoryBankParams" - }, - { - "$ref": "#/components/schemas/GraphMemoryBankParams" - } - ] - }, - "provider_id": { - "type": "string" - }, - "provider_memory_bank_id": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "memory_bank_id", - "params" - ] - }, "RegisterModelRequest": { "type": "object", "properties": { @@ -7993,7 +8404,10 @@ { "$ref": "#/components/schemas/BasicScoringFnParams" } - ] + ], + "discriminator": { + "propertyName": "type" + } } }, "additionalProperties": false, @@ -8090,12 +8504,34 @@ "provider_id" ] }, + "RegisterVectorDbRequest": { + "type": "object", + "properties": { + "vector_db_id": { + "type": "string" + }, + "embedding_model": { + "type": "string" + }, + "embedding_dimension": { + "type": "integer" + }, + "provider_id": { + "type": "string" + }, + "provider_vector_db_id": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "vector_db_id", + "embedding_model" + ] + }, "RunEvalRequest": { "type": "object", "properties": { - "task_id": { - "type": "string" - }, "task_config": { "oneOf": [ { @@ -8104,12 +8540,14 @@ { "$ref": "#/components/schemas/AppEvalTaskConfig" } - ] + ], + "discriminator": { + "propertyName": "type" + } } }, "additionalProperties": false, "required": [ - "task_id", "task_config" ] }, @@ -8254,7 +8692,10 @@ { "$ref": "#/components/schemas/BasicScoringFnParams" } - ] + ], + "discriminator": { + "propertyName": "type" + } }, { "type": "null" @@ -8305,7 +8746,10 @@ { "$ref": "#/components/schemas/BasicScoringFnParams" } - ] + ], + "discriminator": { + "propertyName": "type" + } }, { "type": "null" @@ -8482,7 +8926,10 @@ { "$ref": "#/components/schemas/QATFinetuningConfig" } - ] + ], + "discriminator": { + "propertyName": "type" + } } }, "additionalProperties": false, @@ -8588,54 +9035,6 @@ ], "title": "Response from the synthetic data generation. Batch of (prompt, response, score) tuples that pass the threshold." }, - "UnregisterDatasetRequest": { - "type": "object", - "properties": { - "dataset_id": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "dataset_id" - ] - }, - "UnregisterMemoryBankRequest": { - "type": "object", - "properties": { - "memory_bank_id": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "memory_bank_id" - ] - }, - "UnregisterModelRequest": { - "type": "object", - "properties": { - "model_id": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "model_id" - ] - }, - "UnregisterToolGroupRequest": { - "type": "object", - "properties": { - "tool_group_id": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "tool_group_id" - ] - }, "VersionInfo": { "type": "object", "properties": { @@ -8681,6 +9080,10 @@ "name": "AgentTool", "description": "" }, + { + "name": "AgentTurnInputType", + "description": "" + }, { "name": "AgentTurnResponseEvent", "description": "Streamed agent execution response.\n\n" @@ -8724,6 +9127,10 @@ "name": "AppendRowsRequest", "description": "" }, + { + "name": "ArrayType", + "description": "" + }, { "name": "BasicScoringFnParams", "description": "" @@ -8751,6 +9158,10 @@ "name": "BenchmarkEvalTaskConfig", "description": "" }, + { + "name": "BooleanType", + "description": "" + }, { "name": "BuiltinTool", "description": "" @@ -8759,6 +9170,10 @@ "name": "CancelTrainingJobRequest", "description": "" }, + { + "name": "ChatCompletionInputType", + "description": "" + }, { "name": "ChatCompletionRequest", "description": "" @@ -8783,6 +9198,10 @@ "name": "Checkpoint", "description": "Checkpoint created during training runs\n\n" }, + { + "name": "CompletionInputType", + "description": "" + }, { "name": "CompletionMessage", "description": "" @@ -8827,6 +9246,10 @@ "name": "Dataset", "description": "" }, + { + "name": "DatasetFormat", + "description": "" + }, { "name": "DatasetIO" }, @@ -8834,12 +9257,8 @@ "name": "Datasets" }, { - "name": "DeleteAgentsRequest", - "description": "" - }, - { - "name": "DeleteAgentsSessionRequest", - "description": "" + "name": "DefaultRAGQueryGeneratorConfig", + "description": "" }, { "name": "EfficiencyConfig", @@ -8872,20 +9291,8 @@ "description": "" }, { - "name": "GetAgentsSessionRequest", - "description": "" - }, - { - "name": "GetSpanTreeRequest", - "description": "" - }, - { - "name": "GraphMemoryBank", - "description": "" - }, - { - "name": "GraphMemoryBankParams", - "description": "" + "name": "GreedySamplingStrategy", + "description": "" }, { "name": "HealthInfo", @@ -8895,6 +9302,10 @@ "name": "ImageContentItem", "description": "" }, + { + "name": "ImageDelta", + "description": "" + }, { "name": "Inference" }, @@ -8903,8 +9314,12 @@ "description": "" }, { - "name": "InsertDocumentsRequest", - "description": "" + "name": "InsertChunksRequest", + "description": "" + }, + { + "name": "InsertRequest", + "description": "" }, { "name": "Inspect" @@ -8925,37 +9340,65 @@ "name": "Job", "description": "" }, - { - "name": "JobCancelRequest", - "description": "" - }, { "name": "JobStatus", "description": "" }, { - "name": "KeyValueMemoryBank", - "description": "" - }, - { - "name": "KeyValueMemoryBankParams", - "description": "" - }, - { - "name": "KeywordMemoryBank", - "description": "" - }, - { - "name": "KeywordMemoryBankParams", - "description": "" + "name": "JsonType", + "description": "" }, { "name": "LLMAsJudgeScoringFnParams", "description": "" }, { - "name": "ListRuntimeToolsRequest", - "description": "" + "name": "LLMRAGQueryGeneratorConfig", + "description": "" + }, + { + "name": "ListDatasetsResponse", + "description": "" + }, + { + "name": "ListEvalTasksResponse", + "description": "" + }, + { + "name": "ListModelsResponse", + "description": "" + }, + { + "name": "ListPostTrainingJobsResponse", + "description": "" + }, + { + "name": "ListProvidersResponse", + "description": "" + }, + { + "name": "ListRoutesResponse", + "description": "" + }, + { + "name": "ListScoringFunctionsResponse", + "description": "" + }, + { + "name": "ListShieldsResponse", + "description": "" + }, + { + "name": "ListToolGroupsResponse", + "description": "" + }, + { + "name": "ListToolsResponse", + "description": "" + }, + { + "name": "ListVectorDBsResponse", + "description": "" }, { "name": "LogEventRequest", @@ -8969,16 +9412,6 @@ "name": "LoraFinetuningConfig", "description": "" }, - { - "name": "Memory" - }, - { - "name": "MemoryBankDocument", - "description": "" - }, - { - "name": "MemoryBanks" - }, { "name": "MemoryRetrievalStep", "description": "" @@ -9006,6 +9439,14 @@ { "name": "Models" }, + { + "name": "NumberType", + "description": "" + }, + { + "name": "ObjectType", + "description": "" + }, { "name": "OptimizerConfig", "description": "" @@ -9049,6 +9490,14 @@ "name": "QATFinetuningConfig", "description": "" }, + { + "name": "QueryChunksRequest", + "description": "" + }, + { + "name": "QueryChunksResponse", + "description": "" + }, { "name": "QueryCondition", "description": "" @@ -9058,20 +9507,36 @@ "description": "" }, { - "name": "QueryDocumentsRequest", - "description": "" + "name": "QueryRequest", + "description": "" }, { - "name": "QueryDocumentsResponse", - "description": "" + "name": "QuerySpanTreeResponse", + "description": "" }, { - "name": "QuerySpansRequest", - "description": "" + "name": "QuerySpansResponse", + "description": "" }, { - "name": "QueryTracesRequest", - "description": "" + "name": "QueryTracesResponse", + "description": "" + }, + { + "name": "RAGDocument", + "description": "" + }, + { + "name": "RAGQueryConfig", + "description": "" + }, + { + "name": "RAGQueryGeneratorConfig", + "description": "" + }, + { + "name": "RAGQueryResult", + "description": "" }, { "name": "RegexParserScoringFnParams", @@ -9085,10 +9550,6 @@ "name": "RegisterEvalTaskRequest", "description": "" }, - { - "name": "RegisterMemoryBankRequest", - "description": "" - }, { "name": "RegisterModelRequest", "description": "" @@ -9105,6 +9566,10 @@ "name": "RegisterToolGroupRequest", "description": "" }, + { + "name": "RegisterVectorDbRequest", + "description": "" + }, { "name": "ResponseFormat", "description": "" @@ -9136,10 +9601,6 @@ "name": "SamplingParams", "description": "" }, - { - "name": "SamplingStrategy", - "description": "" - }, { "name": "SaveSpansToDatasetRequest", "description": "" @@ -9213,6 +9674,10 @@ "name": "StopReason", "description": "" }, + { + "name": "StringType", + "description": "" + }, { "name": "StructuredLogEvent", "description": "" @@ -9243,6 +9708,10 @@ "name": "TextContentItem", "description": "" }, + { + "name": "TextDelta", + "description": "" + }, { "name": "TokenLogProbs", "description": "" @@ -9317,6 +9786,14 @@ { "name": "ToolRuntime" }, + { + "name": "TopKSamplingStrategy", + "description": "" + }, + { + "name": "TopPSamplingStrategy", + "description": "" + }, { "name": "Trace", "description": "" @@ -9334,20 +9811,8 @@ "description": "" }, { - "name": "UnregisterDatasetRequest", - "description": "" - }, - { - "name": "UnregisterMemoryBankRequest", - "description": "" - }, - { - "name": "UnregisterModelRequest", - "description": "" - }, - { - "name": "UnregisterToolGroupRequest", - "description": "" + "name": "UnionType", + "description": "" }, { "name": "UnstructuredLogEvent", @@ -9358,12 +9823,14 @@ "description": "" }, { - "name": "VectorMemoryBank", - "description": "" + "name": "VectorDB", + "description": "" }, { - "name": "VectorMemoryBankParams", - "description": "" + "name": "VectorDBs" + }, + { + "name": "VectorIO" }, { "name": "VersionInfo", @@ -9386,8 +9853,6 @@ "EvalTasks", "Inference", "Inspect", - "Memory", - "MemoryBanks", "Models", "PostTraining (Coming Soon)", "Safety", @@ -9397,7 +9862,9 @@ "SyntheticDataGeneration (Coming Soon)", "Telemetry", "ToolGroups", - "ToolRuntime" + "ToolRuntime", + "VectorDBs", + "VectorIO" ] }, { @@ -9409,6 +9876,7 @@ "AgentSessionCreateResponse", "AgentStepResponse", "AgentTool", + "AgentTurnInputType", "AgentTurnResponseEvent", "AgentTurnResponseStepCompletePayload", "AgentTurnResponseStepProgressPayload", @@ -9419,20 +9887,24 @@ "AggregationFunctionType", "AppEvalTaskConfig", "AppendRowsRequest", + "ArrayType", "BasicScoringFnParams", "BatchChatCompletionRequest", "BatchChatCompletionResponse", "BatchCompletionRequest", "BatchCompletionResponse", "BenchmarkEvalTaskConfig", + "BooleanType", "BuiltinTool", "CancelTrainingJobRequest", + "ChatCompletionInputType", "ChatCompletionRequest", "ChatCompletionResponse", "ChatCompletionResponseEvent", "ChatCompletionResponseEventType", "ChatCompletionResponseStreamChunk", "Checkpoint", + "CompletionInputType", "CompletionMessage", "CompletionRequest", "CompletionResponse", @@ -9444,44 +9916,51 @@ "DPOAlignmentConfig", "DataConfig", "Dataset", - "DeleteAgentsRequest", - "DeleteAgentsSessionRequest", + "DatasetFormat", + "DefaultRAGQueryGeneratorConfig", "EfficiencyConfig", "EmbeddingsRequest", "EmbeddingsResponse", "EvalTask", "EvaluateResponse", "EvaluateRowsRequest", - "GetAgentsSessionRequest", - "GetSpanTreeRequest", - "GraphMemoryBank", - "GraphMemoryBankParams", + "GreedySamplingStrategy", "HealthInfo", "ImageContentItem", + "ImageDelta", "InferenceStep", - "InsertDocumentsRequest", + "InsertChunksRequest", + "InsertRequest", "InterleavedContent", "InterleavedContentItem", "InvokeToolRequest", "Job", - "JobCancelRequest", "JobStatus", - "KeyValueMemoryBank", - "KeyValueMemoryBankParams", - "KeywordMemoryBank", - "KeywordMemoryBankParams", + "JsonType", "LLMAsJudgeScoringFnParams", - "ListRuntimeToolsRequest", + "LLMRAGQueryGeneratorConfig", + "ListDatasetsResponse", + "ListEvalTasksResponse", + "ListModelsResponse", + "ListPostTrainingJobsResponse", + "ListProvidersResponse", + "ListRoutesResponse", + "ListScoringFunctionsResponse", + "ListShieldsResponse", + "ListToolGroupsResponse", + "ListToolsResponse", + "ListVectorDBsResponse", "LogEventRequest", "LogSeverity", "LoraFinetuningConfig", - "MemoryBankDocument", "MemoryRetrievalStep", "Message", "MetricEvent", "Model", "ModelCandidate", "ModelType", + "NumberType", + "ObjectType", "OptimizerConfig", "OptimizerType", "PaginatedRowsResult", @@ -9492,20 +9971,26 @@ "PreferenceOptimizeRequest", "ProviderInfo", "QATFinetuningConfig", + "QueryChunksRequest", + "QueryChunksResponse", "QueryCondition", "QueryConditionOp", - "QueryDocumentsRequest", - "QueryDocumentsResponse", - "QuerySpansRequest", - "QueryTracesRequest", + "QueryRequest", + "QuerySpanTreeResponse", + "QuerySpansResponse", + "QueryTracesResponse", + "RAGDocument", + "RAGQueryConfig", + "RAGQueryGeneratorConfig", + "RAGQueryResult", "RegexParserScoringFnParams", "RegisterDatasetRequest", "RegisterEvalTaskRequest", - "RegisterMemoryBankRequest", "RegisterModelRequest", "RegisterScoringFunctionRequest", "RegisterShieldRequest", "RegisterToolGroupRequest", + "RegisterVectorDbRequest", "ResponseFormat", "RouteInfo", "RunEvalRequest", @@ -9513,7 +9998,6 @@ "RunShieldResponse", "SafetyViolation", "SamplingParams", - "SamplingStrategy", "SaveSpansToDatasetRequest", "ScoreBatchRequest", "ScoreBatchResponse", @@ -9530,12 +10014,14 @@ "SpanStatus", "SpanWithStatus", "StopReason", + "StringType", "StructuredLogEvent", "SupervisedFineTuneRequest", "SyntheticDataGenerateRequest", "SyntheticDataGenerationResponse", "SystemMessage", "TextContentItem", + "TextDelta", "TokenLogProbs", "Tool", "ToolCall", @@ -9553,18 +10039,16 @@ "ToolPromptFormat", "ToolResponse", "ToolResponseMessage", + "TopKSamplingStrategy", + "TopPSamplingStrategy", "Trace", "TrainingConfig", "Turn", "URL", - "UnregisterDatasetRequest", - "UnregisterMemoryBankRequest", - "UnregisterModelRequest", - "UnregisterToolGroupRequest", + "UnionType", "UnstructuredLogEvent", "UserMessage", - "VectorMemoryBank", - "VectorMemoryBankParams", + "VectorDB", "VersionInfo", "ViolationLevel" ] diff --git a/docs/resources/llama-stack-spec.yaml b/docs/resources/llama-stack-spec.yaml index 2a573959f..21df2d96f 100644 --- a/docs/resources/llama-stack-spec.yaml +++ b/docs/resources/llama-stack-spec.yaml @@ -76,6 +76,8 @@ components: additionalProperties: false properties: step: + discriminator: + propertyName: step_type oneOf: - $ref: '#/components/schemas/InferenceStep' - $ref: '#/components/schemas/ToolExecutionStep' @@ -105,10 +107,22 @@ components: - name - args type: object + AgentTurnInputType: + additionalProperties: false + properties: + type: + const: agent_turn_input + default: agent_turn_input + type: string + required: + - type + type: object AgentTurnResponseEvent: additionalProperties: false properties: payload: + discriminator: + propertyName: event_type oneOf: - $ref: '#/components/schemas/AgentTurnResponseStepStartPayload' - $ref: '#/components/schemas/AgentTurnResponseStepProgressPayload' @@ -127,6 +141,8 @@ components: default: step_complete type: string step_details: + discriminator: + propertyName: step_type oneOf: - $ref: '#/components/schemas/InferenceStep' - $ref: '#/components/schemas/ToolExecutionStep' @@ -248,6 +264,8 @@ components: additionalProperties: false properties: eval_candidate: + discriminator: + propertyName: type oneOf: - $ref: '#/components/schemas/ModelCandidate' - $ref: '#/components/schemas/AgentCandidate' @@ -255,6 +273,8 @@ components: type: integer scoring_params: additionalProperties: + discriminator: + propertyName: type oneOf: - $ref: '#/components/schemas/LLMAsJudgeScoringFnParams' - $ref: '#/components/schemas/RegexParserScoringFnParams' @@ -290,6 +310,16 @@ components: - dataset_id - rows type: object + ArrayType: + additionalProperties: false + properties: + type: + const: array + default: array + type: string + required: + - type + type: object BasicScoringFnParams: additionalProperties: false properties: @@ -382,6 +412,8 @@ components: additionalProperties: false properties: eval_candidate: + discriminator: + propertyName: type oneOf: - $ref: '#/components/schemas/ModelCandidate' - $ref: '#/components/schemas/AgentCandidate' @@ -395,6 +427,16 @@ components: - type - eval_candidate type: object + BooleanType: + additionalProperties: false + properties: + type: + const: boolean + default: boolean + type: string + required: + - type + type: object BuiltinTool: enum: - brave_search @@ -410,6 +452,16 @@ components: required: - job_uuid type: object + ChatCompletionInputType: + additionalProperties: false + properties: + type: + const: chat_completion_input + default: chat_completion_input + type: string + required: + - type + type: object ChatCompletionRequest: additionalProperties: false properties: @@ -492,6 +544,16 @@ components: type: object Checkpoint: description: Checkpoint created during training runs + CompletionInputType: + additionalProperties: false + properties: + type: + const: completion_input + default: completion_input + type: string + required: + - type + type: object CompletionMessage: additionalProperties: false properties: @@ -569,32 +631,11 @@ components: title: streamed completion response. type: object ContentDelta: + discriminator: + propertyName: type oneOf: - - additionalProperties: false - properties: - text: - type: string - type: - const: text - default: text - type: string - required: - - type - - text - type: object - - additionalProperties: false - properties: - data: - contentEncoding: base64 - type: string - type: - const: image - default: image - type: string - required: - - type - - data - type: object + - $ref: '#/components/schemas/TextDelta' + - $ref: '#/components/schemas/ImageDelta' - $ref: '#/components/schemas/ToolCallDelta' CreateAgentRequest: additionalProperties: false @@ -607,19 +648,14 @@ components: CreateAgentSessionRequest: additionalProperties: false properties: - agent_id: - type: string session_name: type: string required: - - agent_id - session_name type: object CreateAgentTurnRequest: additionalProperties: false properties: - agent_id: - type: string documents: items: additionalProperties: false @@ -645,8 +681,6 @@ components: - $ref: '#/components/schemas/UserMessage' - $ref: '#/components/schemas/ToolResponseMessage' type: array - session_id: - type: string stream: type: boolean toolgroups: @@ -654,8 +688,6 @@ components: $ref: '#/components/schemas/AgentTool' type: array required: - - agent_id - - session_id - messages type: object DPOAlignmentConfig: @@ -680,6 +712,8 @@ components: properties: batch_size: type: integer + data_format: + $ref: '#/components/schemas/DatasetFormat' dataset_id: type: string packed: @@ -696,6 +730,7 @@ components: - dataset_id - batch_size - shuffle + - data_format type: object Dataset: additionalProperties: false @@ -735,24 +770,24 @@ components: - url - metadata type: object - DeleteAgentsRequest: + DatasetFormat: + enum: + - instruct + - dialog + type: string + DefaultRAGQueryGeneratorConfig: additionalProperties: false properties: - agent_id: + separator: + default: ' ' + type: string + type: + const: default + default: default type: string required: - - agent_id - type: object - DeleteAgentsSessionRequest: - additionalProperties: false - properties: - agent_id: - type: string - session_id: - type: string - required: - - agent_id - - session_id + - type + - separator type: object EfficiencyConfig: additionalProperties: false @@ -876,66 +911,25 @@ components: type: string type: array task_config: + discriminator: + propertyName: type oneOf: - $ref: '#/components/schemas/BenchmarkEvalTaskConfig' - $ref: '#/components/schemas/AppEvalTaskConfig' - task_id: - type: string required: - - task_id - input_rows - scoring_functions - task_config type: object - GetAgentsSessionRequest: + GreedySamplingStrategy: additionalProperties: false properties: - turn_ids: - items: - type: string - type: array - type: object - GetSpanTreeRequest: - additionalProperties: false - properties: - attributes_to_return: - items: - type: string - type: array - type: object - GraphMemoryBank: - additionalProperties: false - properties: - identifier: - type: string - memory_bank_type: - const: graph - default: graph - type: string - provider_id: - type: string - provider_resource_id: - type: string type: - const: memory_bank - default: memory_bank + const: greedy + default: greedy type: string required: - - identifier - - provider_resource_id - - provider_id - type - - memory_bank_type - type: object - GraphMemoryBankParams: - additionalProperties: false - properties: - memory_bank_type: - const: graph - default: graph - type: string - required: - - memory_bank_type type: object HealthInfo: additionalProperties: false @@ -948,17 +942,36 @@ components: ImageContentItem: additionalProperties: false properties: - data: + image: + additionalProperties: false + properties: + data: + contentEncoding: base64 + type: string + url: + $ref: '#/components/schemas/URL' + type: object + type: + const: image + default: image + type: string + required: + - type + - image + type: object + ImageDelta: + additionalProperties: false + properties: + image: contentEncoding: base64 type: string type: const: image default: image type: string - url: - $ref: '#/components/schemas/URL' required: - type + - image type: object InferenceStep: additionalProperties: false @@ -985,20 +998,53 @@ components: - step_type - model_response type: object - InsertDocumentsRequest: + InsertChunksRequest: additionalProperties: false properties: - bank_id: - type: string - documents: + chunks: items: - $ref: '#/components/schemas/MemoryBankDocument' + additionalProperties: false + properties: + content: + $ref: '#/components/schemas/InterleavedContent' + metadata: + additionalProperties: + oneOf: + - type: 'null' + - type: boolean + - type: number + - type: string + - type: array + - type: object + type: object + required: + - content + - metadata + type: object type: array ttl_seconds: type: integer + vector_db_id: + type: string + required: + - vector_db_id + - chunks + type: object + InsertRequest: + additionalProperties: false + properties: + chunk_size_in_tokens: + type: integer + documents: + items: + $ref: '#/components/schemas/RAGDocument' + type: array + vector_db_id: + type: string required: - - bank_id - documents + - vector_db_id + - chunk_size_in_tokens type: object InterleavedContent: oneOf: @@ -1008,13 +1054,15 @@ components: $ref: '#/components/schemas/InterleavedContentItem' type: array InterleavedContentItem: + discriminator: + propertyName: type oneOf: - $ref: '#/components/schemas/ImageContentItem' - $ref: '#/components/schemas/TextContentItem' InvokeToolRequest: additionalProperties: false properties: - args: + kwargs: additionalProperties: oneOf: - type: 'null' @@ -1028,7 +1076,7 @@ components: type: string required: - tool_name - - args + - kwargs type: object Job: additionalProperties: false @@ -1038,17 +1086,6 @@ components: required: - job_id type: object - JobCancelRequest: - additionalProperties: false - properties: - job_id: - type: string - task_id: - type: string - required: - - task_id - - job_id - type: object JobStatus: enum: - completed @@ -1056,73 +1093,15 @@ components: - failed - scheduled type: string - KeyValueMemoryBank: + JsonType: additionalProperties: false properties: - identifier: - type: string - memory_bank_type: - const: keyvalue - default: keyvalue - type: string - provider_id: - type: string - provider_resource_id: - type: string type: - const: memory_bank - default: memory_bank + const: json + default: json type: string required: - - identifier - - provider_resource_id - - provider_id - type - - memory_bank_type - type: object - KeyValueMemoryBankParams: - additionalProperties: false - properties: - memory_bank_type: - const: keyvalue - default: keyvalue - type: string - required: - - memory_bank_type - type: object - KeywordMemoryBank: - additionalProperties: false - properties: - identifier: - type: string - memory_bank_type: - const: keyword - default: keyword - type: string - provider_id: - type: string - provider_resource_id: - type: string - type: - const: memory_bank - default: memory_bank - type: string - required: - - identifier - - provider_resource_id - - provider_id - - type - - memory_bank_type - type: object - KeywordMemoryBankParams: - additionalProperties: false - properties: - memory_bank_type: - const: keyword - default: keyword - type: string - required: - - memory_bank_type type: object LLMAsJudgeScoringFnParams: additionalProperties: false @@ -1147,16 +1126,144 @@ components: - type - judge_model type: object - ListRuntimeToolsRequest: + LLMRAGQueryGeneratorConfig: additionalProperties: false properties: - mcp_endpoint: - $ref: '#/components/schemas/URL' + model: + type: string + template: + type: string + type: + const: llm + default: llm + type: string + required: + - type + - model + - template + type: object + ListDatasetsResponse: + additionalProperties: false + properties: + data: + items: + $ref: '#/components/schemas/Dataset' + type: array + required: + - data + type: object + ListEvalTasksResponse: + additionalProperties: false + properties: + data: + items: + $ref: '#/components/schemas/EvalTask' + type: array + required: + - data + type: object + ListModelsResponse: + additionalProperties: false + properties: + data: + items: + $ref: '#/components/schemas/Model' + type: array + required: + - data + type: object + ListPostTrainingJobsResponse: + additionalProperties: false + properties: + data: + items: + additionalProperties: false + properties: + job_uuid: + type: string + required: + - job_uuid + type: object + type: array + required: + - data + type: object + ListProvidersResponse: + additionalProperties: false + properties: + data: + items: + $ref: '#/components/schemas/ProviderInfo' + type: array + required: + - data + type: object + ListRoutesResponse: + additionalProperties: false + properties: + data: + items: + $ref: '#/components/schemas/RouteInfo' + type: array + required: + - data + type: object + ListScoringFunctionsResponse: + additionalProperties: false + properties: + data: + items: + $ref: '#/components/schemas/ScoringFn' + type: array + required: + - data + type: object + ListShieldsResponse: + additionalProperties: false + properties: + data: + items: + $ref: '#/components/schemas/Shield' + type: array + required: + - data + type: object + ListToolGroupsResponse: + additionalProperties: false + properties: + data: + items: + $ref: '#/components/schemas/ToolGroup' + type: array + required: + - data + type: object + ListToolsResponse: + additionalProperties: false + properties: + data: + items: + $ref: '#/components/schemas/Tool' + type: array + required: + - data + type: object + ListVectorDBsResponse: + additionalProperties: false + properties: + data: + items: + $ref: '#/components/schemas/VectorDB' + type: array + required: + - data type: object LogEventRequest: additionalProperties: false properties: event: + discriminator: + propertyName: type oneOf: - $ref: '#/components/schemas/UnstructuredLogEvent' - $ref: '#/components/schemas/MetricEvent' @@ -1209,36 +1316,6 @@ components: - rank - alpha type: object - MemoryBankDocument: - additionalProperties: false - properties: - content: - oneOf: - - type: string - - $ref: '#/components/schemas/InterleavedContentItem' - - items: - $ref: '#/components/schemas/InterleavedContentItem' - type: array - - $ref: '#/components/schemas/URL' - document_id: - type: string - metadata: - additionalProperties: - oneOf: - - type: 'null' - - type: boolean - - type: number - - type: string - - type: array - - type: object - type: object - mime_type: - type: string - required: - - document_id - - content - - metadata - type: object MemoryRetrievalStep: additionalProperties: false properties: @@ -1247,10 +1324,6 @@ components: type: string inserted_context: $ref: '#/components/schemas/InterleavedContent' - memory_bank_ids: - items: - type: string - type: array started_at: format: date-time type: string @@ -1262,14 +1335,18 @@ components: type: string turn_id: type: string + vector_db_ids: + type: string required: - turn_id - step_id - step_type - - memory_bank_ids + - vector_db_ids - inserted_context type: object Message: + discriminator: + propertyName: role oneOf: - $ref: '#/components/schemas/UserMessage' - $ref: '#/components/schemas/SystemMessage' @@ -1373,6 +1450,26 @@ components: - llm - embedding type: string + NumberType: + additionalProperties: false + properties: + type: + const: number + default: number + type: string + required: + - type + type: object + ObjectType: + additionalProperties: false + properties: + type: + const: object + default: object + type: string + required: + - type + type: object OptimizerConfig: additionalProperties: false properties: @@ -1420,97 +1517,19 @@ components: - total_count type: object ParamType: + discriminator: + propertyName: type oneOf: - - additionalProperties: false - properties: - type: - const: string - default: string - type: string - required: - - type - type: object - - additionalProperties: false - properties: - type: - const: number - default: number - type: string - required: - - type - type: object - - additionalProperties: false - properties: - type: - const: boolean - default: boolean - type: string - required: - - type - type: object - - additionalProperties: false - properties: - type: - const: array - default: array - type: string - required: - - type - type: object - - additionalProperties: false - properties: - type: - const: object - default: object - type: string - required: - - type - type: object - - additionalProperties: false - properties: - type: - const: json - default: json - type: string - required: - - type - type: object - - additionalProperties: false - properties: - type: - const: union - default: union - type: string - required: - - type - type: object - - additionalProperties: false - properties: - type: - const: chat_completion_input - default: chat_completion_input - type: string - required: - - type - type: object - - additionalProperties: false - properties: - type: - const: completion_input - default: completion_input - type: string - required: - - type - type: object - - additionalProperties: false - properties: - type: - const: agent_turn_input - default: agent_turn_input - type: string - required: - - type - type: object + - $ref: '#/components/schemas/StringType' + - $ref: '#/components/schemas/NumberType' + - $ref: '#/components/schemas/BooleanType' + - $ref: '#/components/schemas/ArrayType' + - $ref: '#/components/schemas/ObjectType' + - $ref: '#/components/schemas/JsonType' + - $ref: '#/components/schemas/UnionType' + - $ref: '#/components/schemas/ChatCompletionInputType' + - $ref: '#/components/schemas/CompletionInputType' + - $ref: '#/components/schemas/AgentTurnInputType' PostTrainingJob: additionalProperties: false properties: @@ -1611,11 +1630,14 @@ components: ProviderInfo: additionalProperties: false properties: + api: + type: string provider_id: type: string provider_type: type: string required: + - api - provider_id - provider_type type: object @@ -1635,6 +1657,59 @@ components: - quantizer_name - group_size type: object + QueryChunksRequest: + additionalProperties: false + properties: + params: + additionalProperties: + oneOf: + - type: 'null' + - type: boolean + - type: number + - type: string + - type: array + - type: object + type: object + query: + $ref: '#/components/schemas/InterleavedContent' + vector_db_id: + type: string + required: + - vector_db_id + - query + type: object + QueryChunksResponse: + additionalProperties: false + properties: + chunks: + items: + additionalProperties: false + properties: + content: + $ref: '#/components/schemas/InterleavedContent' + metadata: + additionalProperties: + oneOf: + - type: 'null' + - type: boolean + - type: number + - type: string + - type: array + - type: object + type: object + required: + - content + - metadata + type: object + type: array + scores: + items: + type: number + type: array + required: + - chunks + - scores + type: object QueryCondition: additionalProperties: false properties: @@ -1662,12 +1737,65 @@ components: - gt - lt type: string - QueryDocumentsRequest: + QueryRequest: additionalProperties: false properties: - bank_id: + content: + $ref: '#/components/schemas/InterleavedContent' + query_config: + $ref: '#/components/schemas/RAGQueryConfig' + vector_db_ids: + items: + type: string + type: array + required: + - content + - vector_db_ids + type: object + QuerySpanTreeResponse: + additionalProperties: false + properties: + data: + additionalProperties: + $ref: '#/components/schemas/SpanWithStatus' + type: object + required: + - data + type: object + QuerySpansResponse: + additionalProperties: false + properties: + data: + items: + $ref: '#/components/schemas/Span' + type: array + required: + - data + type: object + QueryTracesResponse: + additionalProperties: false + properties: + data: + items: + $ref: '#/components/schemas/Trace' + type: array + required: + - data + type: object + RAGDocument: + additionalProperties: false + properties: + content: + oneOf: + - type: string + - $ref: '#/components/schemas/InterleavedContentItem' + - items: + $ref: '#/components/schemas/InterleavedContentItem' + type: array + - $ref: '#/components/schemas/URL' + document_id: type: string - params: + metadata: additionalProperties: oneOf: - type: 'null' @@ -1677,71 +1805,40 @@ components: - type: array - type: object type: object - query: + mime_type: + type: string + required: + - document_id + - content + - metadata + type: object + RAGQueryConfig: + additionalProperties: false + properties: + max_chunks: + default: 5 + type: integer + max_tokens_in_context: + default: 4096 + type: integer + query_generator_config: + $ref: '#/components/schemas/RAGQueryGeneratorConfig' + required: + - query_generator_config + - max_tokens_in_context + - max_chunks + type: object + RAGQueryGeneratorConfig: + discriminator: + propertyName: type + oneOf: + - $ref: '#/components/schemas/DefaultRAGQueryGeneratorConfig' + - $ref: '#/components/schemas/LLMRAGQueryGeneratorConfig' + RAGQueryResult: + additionalProperties: false + properties: + content: $ref: '#/components/schemas/InterleavedContent' - required: - - bank_id - - query - type: object - QueryDocumentsResponse: - additionalProperties: false - properties: - chunks: - items: - additionalProperties: false - properties: - content: - $ref: '#/components/schemas/InterleavedContent' - document_id: - type: string - token_count: - type: integer - required: - - content - - token_count - - document_id - type: object - type: array - scores: - items: - type: number - type: array - required: - - chunks - - scores - type: object - QuerySpansRequest: - additionalProperties: false - properties: - attribute_filters: - items: - $ref: '#/components/schemas/QueryCondition' - type: array - attributes_to_return: - items: - type: string - type: array - max_depth: - type: integer - required: - - attribute_filters - - attributes_to_return - type: object - QueryTracesRequest: - additionalProperties: false - properties: - attribute_filters: - items: - $ref: '#/components/schemas/QueryCondition' - type: array - limit: - type: integer - offset: - type: integer - order_by: - items: - type: string - type: array type: object RegexParserScoringFnParams: additionalProperties: false @@ -1821,25 +1918,6 @@ components: - dataset_id - scoring_functions type: object - RegisterMemoryBankRequest: - additionalProperties: false - properties: - memory_bank_id: - type: string - params: - oneOf: - - $ref: '#/components/schemas/VectorMemoryBankParams' - - $ref: '#/components/schemas/KeyValueMemoryBankParams' - - $ref: '#/components/schemas/KeywordMemoryBankParams' - - $ref: '#/components/schemas/GraphMemoryBankParams' - provider_id: - type: string - provider_memory_bank_id: - type: string - required: - - memory_bank_id - - params - type: object RegisterModelRequest: additionalProperties: false properties: @@ -1870,6 +1948,8 @@ components: description: type: string params: + discriminator: + propertyName: type oneOf: - $ref: '#/components/schemas/LLMAsJudgeScoringFnParams' - $ref: '#/components/schemas/RegexParserScoringFnParams' @@ -1932,7 +2012,26 @@ components: - toolgroup_id - provider_id type: object + RegisterVectorDbRequest: + additionalProperties: false + properties: + embedding_dimension: + type: integer + embedding_model: + type: string + provider_id: + type: string + provider_vector_db_id: + type: string + vector_db_id: + type: string + required: + - vector_db_id + - embedding_model + type: object ResponseFormat: + discriminator: + propertyName: type oneOf: - additionalProperties: false properties: @@ -1994,13 +2093,12 @@ components: additionalProperties: false properties: task_config: + discriminator: + propertyName: type oneOf: - $ref: '#/components/schemas/BenchmarkEvalTaskConfig' - $ref: '#/components/schemas/AppEvalTaskConfig' - task_id: - type: string required: - - task_id - task_config type: object RunShieldRequest: @@ -2064,26 +2162,15 @@ components: default: 1.0 type: number strategy: - $ref: '#/components/schemas/SamplingStrategy' - default: greedy - temperature: - default: 0.0 - type: number - top_k: - default: 0 - type: integer - top_p: - default: 0.95 - type: number + discriminator: + propertyName: type + oneOf: + - $ref: '#/components/schemas/GreedySamplingStrategy' + - $ref: '#/components/schemas/TopPSamplingStrategy' + - $ref: '#/components/schemas/TopKSamplingStrategy' required: - strategy type: object - SamplingStrategy: - enum: - - greedy - - top_p - - top_k - type: string SaveSpansToDatasetRequest: additionalProperties: false properties: @@ -2114,7 +2201,9 @@ components: scoring_functions: additionalProperties: oneOf: - - oneOf: + - discriminator: + propertyName: type + oneOf: - $ref: '#/components/schemas/LLMAsJudgeScoringFnParams' - $ref: '#/components/schemas/RegexParserScoringFnParams' - $ref: '#/components/schemas/BasicScoringFnParams' @@ -2155,7 +2244,9 @@ components: scoring_functions: additionalProperties: oneOf: - - oneOf: + - discriminator: + propertyName: type + oneOf: - $ref: '#/components/schemas/LLMAsJudgeScoringFnParams' - $ref: '#/components/schemas/RegexParserScoringFnParams' - $ref: '#/components/schemas/BasicScoringFnParams' @@ -2193,6 +2284,8 @@ components: - type: object type: object params: + discriminator: + propertyName: type oneOf: - $ref: '#/components/schemas/LLMAsJudgeScoringFnParams' - $ref: '#/components/schemas/RegexParserScoringFnParams' @@ -2247,12 +2340,6 @@ components: Session: additionalProperties: false properties: - memory_bank: - oneOf: - - $ref: '#/components/schemas/VectorMemoryBank' - - $ref: '#/components/schemas/KeyValueMemoryBank' - - $ref: '#/components/schemas/KeywordMemoryBank' - - $ref: '#/components/schemas/GraphMemoryBank' session_id: type: string session_name: @@ -2432,6 +2519,16 @@ components: - end_of_message - out_of_tokens type: string + StringType: + additionalProperties: false + properties: + type: + const: string + default: string + type: string + required: + - type + type: object StructuredLogEvent: additionalProperties: false properties: @@ -2446,6 +2543,8 @@ components: - type: object type: object payload: + discriminator: + propertyName: type oneOf: - $ref: '#/components/schemas/SpanStartPayload' - $ref: '#/components/schemas/SpanEndPayload' @@ -2471,6 +2570,8 @@ components: additionalProperties: false properties: algorithm_config: + discriminator: + propertyName: type oneOf: - $ref: '#/components/schemas/LoraFinetuningConfig' - $ref: '#/components/schemas/QATFinetuningConfig' @@ -2588,6 +2689,19 @@ components: - type - text type: object + TextDelta: + additionalProperties: false + properties: + text: + type: string + type: + const: text + default: text + type: string + required: + - type + - text + type: object TokenLogProbs: additionalProperties: false properties: @@ -2683,19 +2797,19 @@ components: ToolCallDelta: additionalProperties: false properties: - content: + parse_status: + $ref: '#/components/schemas/ToolCallParseStatus' + tool_call: oneOf: - type: string - $ref: '#/components/schemas/ToolCall' - parse_status: - $ref: '#/components/schemas/ToolCallParseStatus' type: const: tool_call default: tool_call type: string required: - type - - content + - tool_call - parse_status type: object ToolCallParseStatus: @@ -2931,6 +3045,34 @@ components: - tool_name - content type: object + TopKSamplingStrategy: + additionalProperties: false + properties: + top_k: + type: integer + type: + const: top_k + default: top_k + type: string + required: + - type + - top_k + type: object + TopPSamplingStrategy: + additionalProperties: false + properties: + temperature: + type: number + top_p: + default: 0.95 + type: number + type: + const: top_p + default: top_p + type: string + required: + - type + type: object Trace: additionalProperties: false properties: @@ -3017,6 +3159,8 @@ components: type: string steps: items: + discriminator: + propertyName: step_type oneOf: - $ref: '#/components/schemas/InferenceStep' - $ref: '#/components/schemas/ToolExecutionStep' @@ -3043,37 +3187,15 @@ components: required: - uri type: object - UnregisterDatasetRequest: + UnionType: additionalProperties: false properties: - dataset_id: + type: + const: union + default: union type: string required: - - dataset_id - type: object - UnregisterMemoryBankRequest: - additionalProperties: false - properties: - memory_bank_id: - type: string - required: - - memory_bank_id - type: object - UnregisterModelRequest: - additionalProperties: false - properties: - model_id: - type: string - required: - - model_id - type: object - UnregisterToolGroupRequest: - additionalProperties: false - properties: - tool_group_id: - type: string - required: - - tool_group_id + - type type: object UnstructuredLogEvent: additionalProperties: false @@ -3126,58 +3248,30 @@ components: - role - content type: object - VectorMemoryBank: + VectorDB: additionalProperties: false properties: - chunk_size_in_tokens: - type: integer embedding_dimension: - default: 384 type: integer embedding_model: type: string identifier: type: string - memory_bank_type: - const: vector - default: vector - type: string - overlap_size_in_tokens: - type: integer provider_id: type: string provider_resource_id: type: string type: - const: memory_bank - default: memory_bank + const: vector_db + default: vector_db type: string required: - identifier - provider_resource_id - provider_id - type - - memory_bank_type - embedding_model - - chunk_size_in_tokens - type: object - VectorMemoryBankParams: - additionalProperties: false - properties: - chunk_size_in_tokens: - type: integer - embedding_model: - type: string - memory_bank_type: - const: vector - default: vector - type: string - overlap_size_in_tokens: - type: integer - required: - - memory_bank_type - - embedding_model - - chunk_size_in_tokens + - embedding_dimension type: object VersionInfo: additionalProperties: false @@ -3198,11 +3292,11 @@ info: \ a set of endpoints and their corresponding interfaces that are tailored\ \ to\n best leverage Llama Models." title: Llama Stack Specification - version: alpha + version: v1 jsonSchemaDialect: https://json-schema.org/draft/2020-12/schema openapi: 3.1.0 paths: - /alpha/agents/create: + /v1/agents: post: parameters: - description: JSON-encoded provider data which will be made available to the @@ -3234,9 +3328,14 @@ paths: description: OK tags: - Agents - /alpha/agents/delete: - post: + /v1/agents/{agent_id}: + delete: parameters: + - in: path + name: agent_id + required: true + schema: + type: string - description: JSON-encoded provider data which will be made available to the adapter servicing the API in: header @@ -3251,20 +3350,19 @@ paths: required: false schema: type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/DeleteAgentsRequest' - required: true responses: '200': description: OK tags: - Agents - /alpha/agents/session/create: + /v1/agents/{agent_id}/session: post: parameters: + - in: path + name: agent_id + required: true + schema: + type: string - description: JSON-encoded provider data which will be made available to the adapter servicing the API in: header @@ -3294,9 +3392,19 @@ paths: description: OK tags: - Agents - /alpha/agents/session/delete: - post: + /v1/agents/{agent_id}/session/{session_id}: + delete: parameters: + - in: path + name: session_id + required: true + schema: + type: string + - in: path + name: agent_id + required: true + schema: + type: string - description: JSON-encoded provider data which will be made available to the adapter servicing the API in: header @@ -3311,30 +3419,30 @@ paths: required: false schema: type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/DeleteAgentsSessionRequest' - required: true responses: '200': description: OK tags: - Agents - /alpha/agents/session/get: - post: + get: parameters: - - in: query + - in: path + name: session_id + required: true + schema: + type: string + - in: path name: agent_id required: true schema: type: string - in: query - name: session_id - required: true + name: turn_ids + required: false schema: - type: string + items: + type: string + type: array - description: JSON-encoded provider data which will be made available to the adapter servicing the API in: header @@ -3349,12 +3457,6 @@ paths: required: false schema: type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/GetAgentsSessionRequest' - required: true responses: '200': content: @@ -3364,55 +3466,19 @@ paths: description: OK tags: - Agents - /alpha/agents/step/get: - get: + /v1/agents/{agent_id}/session/{session_id}/turn: + post: parameters: - - in: query + - in: path name: agent_id required: true schema: type: string - - in: query + - in: path name: session_id required: true schema: type: string - - in: query - name: turn_id - required: true - schema: - type: string - - in: query - name: step_id - required: true - schema: - type: string - - description: JSON-encoded provider data which will be made available to the - adapter servicing the API - in: header - name: X-LlamaStack-Provider-Data - required: false - schema: - type: string - - description: Version of the client making the request. This is used to ensure - that the client and server are compatible. - in: header - name: X-LlamaStack-Client-Version - required: false - schema: - type: string - responses: - '200': - content: - application/json: - schema: - $ref: '#/components/schemas/AgentStepResponse' - description: OK - tags: - - Agents - /alpha/agents/turn/create: - post: - parameters: - description: JSON-encoded provider data which will be made available to the adapter servicing the API in: header @@ -3445,20 +3511,20 @@ paths: streamed agent turn completion response. tags: - Agents - /alpha/agents/turn/get: + /v1/agents/{agent_id}/session/{session_id}/turn/{turn_id}: get: parameters: - - in: query + - in: path name: agent_id required: true schema: type: string - - in: query + - in: path name: session_id required: true schema: type: string - - in: query + - in: path name: turn_id required: true schema: @@ -3486,7 +3552,53 @@ paths: description: OK tags: - Agents - /alpha/batch-inference/chat-completion: + /v1/agents/{agent_id}/session/{session_id}/turn/{turn_id}/step/{step_id}: + get: + parameters: + - in: path + name: agent_id + required: true + schema: + type: string + - in: path + name: session_id + required: true + schema: + type: string + - in: path + name: turn_id + required: true + schema: + type: string + - in: path + name: step_id + required: true + schema: + type: string + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-Provider-Data + required: false + schema: + type: string + - description: Version of the client making the request. This is used to ensure + that the client and server are compatible. + in: header + name: X-LlamaStack-Client-Version + required: false + schema: + type: string + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/AgentStepResponse' + description: OK + tags: + - Agents + /v1/batch-inference/chat-completion: post: parameters: - description: JSON-encoded provider data which will be made available to the @@ -3518,7 +3630,7 @@ paths: description: OK tags: - BatchInference (Coming Soon) - /alpha/batch-inference/completion: + /v1/batch-inference/completion: post: parameters: - description: JSON-encoded provider data which will be made available to the @@ -3550,35 +3662,7 @@ paths: description: OK tags: - BatchInference (Coming Soon) - /alpha/datasetio/append-rows: - post: - parameters: - - description: JSON-encoded provider data which will be made available to the - adapter servicing the API - in: header - name: X-LlamaStack-Provider-Data - required: false - schema: - type: string - - description: Version of the client making the request. This is used to ensure - that the client and server are compatible. - in: header - name: X-LlamaStack-Client-Version - required: false - schema: - type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/AppendRowsRequest' - required: true - responses: - '200': - description: OK - tags: - - DatasetIO - /alpha/datasetio/get-rows-paginated: + /v1/datasetio/rows: get: parameters: - in: query @@ -3624,10 +3708,116 @@ paths: description: OK tags: - DatasetIO - /alpha/datasets/get: + post: + parameters: + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-Provider-Data + required: false + schema: + type: string + - description: Version of the client making the request. This is used to ensure + that the client and server are compatible. + in: header + name: X-LlamaStack-Client-Version + required: false + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/AppendRowsRequest' + required: true + responses: + '200': + description: OK + tags: + - DatasetIO + /v1/datasets: get: parameters: - - in: query + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-Provider-Data + required: false + schema: + type: string + - description: Version of the client making the request. This is used to ensure + that the client and server are compatible. + in: header + name: X-LlamaStack-Client-Version + required: false + schema: + type: string + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/ListDatasetsResponse' + description: OK + tags: + - Datasets + post: + parameters: + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-Provider-Data + required: false + schema: + type: string + - description: Version of the client making the request. This is used to ensure + that the client and server are compatible. + in: header + name: X-LlamaStack-Client-Version + required: false + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/RegisterDatasetRequest' + required: true + responses: + '200': + description: OK + tags: + - Datasets + /v1/datasets/{dataset_id}: + delete: + parameters: + - in: path + name: dataset_id + required: true + schema: + type: string + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-Provider-Data + required: false + schema: + type: string + - description: Version of the client making the request. This is used to ensure + that the client and server are compatible. + in: header + name: X-LlamaStack-Client-Version + required: false + schema: + type: string + responses: + '200': + description: OK + tags: + - Datasets + get: + parameters: + - in: path name: dataset_id required: true schema: @@ -3657,7 +3847,7 @@ paths: description: OK tags: - Datasets - /alpha/datasets/list: + /v1/eval-tasks: get: parameters: - description: JSON-encoded provider data which will be made available to the @@ -3677,13 +3867,12 @@ paths: responses: '200': content: - application/jsonl: + application/json: schema: - $ref: '#/components/schemas/Dataset' + $ref: '#/components/schemas/ListEvalTasksResponse' description: OK tags: - - Datasets - /alpha/datasets/register: + - EvalTasks post: parameters: - description: JSON-encoded provider data which will be made available to the @@ -3704,46 +3893,18 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/RegisterDatasetRequest' + $ref: '#/components/schemas/RegisterEvalTaskRequest' required: true responses: '200': description: OK tags: - - Datasets - /alpha/datasets/unregister: - post: - parameters: - - description: JSON-encoded provider data which will be made available to the - adapter servicing the API - in: header - name: X-LlamaStack-Provider-Data - required: false - schema: - type: string - - description: Version of the client making the request. This is used to ensure - that the client and server are compatible. - in: header - name: X-LlamaStack-Client-Version - required: false - schema: - type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/UnregisterDatasetRequest' - required: true - responses: - '200': - description: OK - tags: - - Datasets - /alpha/eval-tasks/get: + - EvalTasks + /v1/eval-tasks/{eval_task_id}: get: parameters: - - in: query - name: name + - in: path + name: eval_task_id required: true schema: type: string @@ -3772,63 +3933,14 @@ paths: description: OK tags: - EvalTasks - /alpha/eval-tasks/list: - get: - parameters: - - description: JSON-encoded provider data which will be made available to the - adapter servicing the API - in: header - name: X-LlamaStack-Provider-Data - required: false - schema: - type: string - - description: Version of the client making the request. This is used to ensure - that the client and server are compatible. - in: header - name: X-LlamaStack-Client-Version - required: false - schema: - type: string - responses: - '200': - content: - application/jsonl: - schema: - $ref: '#/components/schemas/EvalTask' - description: OK - tags: - - EvalTasks - /alpha/eval-tasks/register: + /v1/eval/tasks/{task_id}/evaluations: post: parameters: - - description: JSON-encoded provider data which will be made available to the - adapter servicing the API - in: header - name: X-LlamaStack-Provider-Data - required: false - schema: - type: string - - description: Version of the client making the request. This is used to ensure - that the client and server are compatible. - in: header - name: X-LlamaStack-Client-Version - required: false - schema: - type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/RegisterEvalTaskRequest' + - in: path + name: task_id required: true - responses: - '200': - description: OK - tags: - - EvalTasks - /alpha/eval/evaluate-rows: - post: - parameters: + schema: + type: string - description: JSON-encoded provider data which will be made available to the adapter servicing the API in: header @@ -3858,111 +3970,14 @@ paths: description: OK tags: - Eval - /alpha/eval/job/cancel: + /v1/eval/tasks/{task_id}/jobs: post: parameters: - - description: JSON-encoded provider data which will be made available to the - adapter servicing the API - in: header - name: X-LlamaStack-Provider-Data - required: false - schema: - type: string - - description: Version of the client making the request. This is used to ensure - that the client and server are compatible. - in: header - name: X-LlamaStack-Client-Version - required: false - schema: - type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/JobCancelRequest' - required: true - responses: - '200': - description: OK - tags: - - Eval - /alpha/eval/job/result: - get: - parameters: - - in: query + - in: path name: task_id required: true schema: type: string - - in: query - name: job_id - required: true - schema: - type: string - - description: JSON-encoded provider data which will be made available to the - adapter servicing the API - in: header - name: X-LlamaStack-Provider-Data - required: false - schema: - type: string - - description: Version of the client making the request. This is used to ensure - that the client and server are compatible. - in: header - name: X-LlamaStack-Client-Version - required: false - schema: - type: string - responses: - '200': - content: - application/json: - schema: - $ref: '#/components/schemas/EvaluateResponse' - description: OK - tags: - - Eval - /alpha/eval/job/status: - get: - parameters: - - in: query - name: task_id - required: true - schema: - type: string - - in: query - name: job_id - required: true - schema: - type: string - - description: JSON-encoded provider data which will be made available to the - adapter servicing the API - in: header - name: X-LlamaStack-Provider-Data - required: false - schema: - type: string - - description: Version of the client making the request. This is used to ensure - that the client and server are compatible. - in: header - name: X-LlamaStack-Client-Version - required: false - schema: - type: string - responses: - '200': - content: - application/json: - schema: - oneOf: - - $ref: '#/components/schemas/JobStatus' - - type: 'null' - description: OK - tags: - - Eval - /alpha/eval/run-eval: - post: - parameters: - description: JSON-encoded provider data which will be made available to the adapter servicing the API in: header @@ -3992,7 +4007,112 @@ paths: description: OK tags: - Eval - /alpha/health: + /v1/eval/tasks/{task_id}/jobs/{job_id}: + delete: + parameters: + - in: path + name: task_id + required: true + schema: + type: string + - in: path + name: job_id + required: true + schema: + type: string + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-Provider-Data + required: false + schema: + type: string + - description: Version of the client making the request. This is used to ensure + that the client and server are compatible. + in: header + name: X-LlamaStack-Client-Version + required: false + schema: + type: string + responses: + '200': + description: OK + tags: + - Eval + get: + parameters: + - in: path + name: task_id + required: true + schema: + type: string + - in: path + name: job_id + required: true + schema: + type: string + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-Provider-Data + required: false + schema: + type: string + - description: Version of the client making the request. This is used to ensure + that the client and server are compatible. + in: header + name: X-LlamaStack-Client-Version + required: false + schema: + type: string + responses: + '200': + content: + application/json: + schema: + oneOf: + - $ref: '#/components/schemas/JobStatus' + - type: 'null' + description: OK + tags: + - Eval + /v1/eval/tasks/{task_id}/jobs/{job_id}/result: + get: + parameters: + - in: path + name: job_id + required: true + schema: + type: string + - in: path + name: task_id + required: true + schema: + type: string + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-Provider-Data + required: false + schema: + type: string + - description: Version of the client making the request. This is used to ensure + that the client and server are compatible. + in: header + name: X-LlamaStack-Client-Version + required: false + schema: + type: string + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluateResponse' + description: OK + tags: + - Eval + /v1/health: get: parameters: - description: JSON-encoded provider data which will be made available to the @@ -4018,7 +4138,7 @@ paths: description: OK tags: - Inspect - /alpha/inference/chat-completion: + /v1/inference/chat-completion: post: parameters: - description: JSON-encoded provider data which will be made available to the @@ -4052,7 +4172,7 @@ paths: description: Chat completion response. **OR** SSE-stream of these events. tags: - Inference - /alpha/inference/completion: + /v1/inference/completion: post: parameters: - description: JSON-encoded provider data which will be made available to the @@ -4086,7 +4206,7 @@ paths: description: Completion response. **OR** streamed completion response. tags: - Inference - /alpha/inference/embeddings: + /v1/inference/embeddings: post: parameters: - description: JSON-encoded provider data which will be made available to the @@ -4118,44 +4238,7 @@ paths: description: OK tags: - Inference - /alpha/memory-banks/get: - get: - parameters: - - in: query - name: memory_bank_id - required: true - schema: - type: string - - description: JSON-encoded provider data which will be made available to the - adapter servicing the API - in: header - name: X-LlamaStack-Provider-Data - required: false - schema: - type: string - - description: Version of the client making the request. This is used to ensure - that the client and server are compatible. - in: header - name: X-LlamaStack-Client-Version - required: false - schema: - type: string - responses: - '200': - content: - application/json: - schema: - oneOf: - - oneOf: - - $ref: '#/components/schemas/VectorMemoryBank' - - $ref: '#/components/schemas/KeyValueMemoryBank' - - $ref: '#/components/schemas/KeywordMemoryBank' - - $ref: '#/components/schemas/GraphMemoryBank' - - type: 'null' - description: OK - tags: - - MemoryBanks - /alpha/memory-banks/list: + /v1/inspect/providers: get: parameters: - description: JSON-encoded provider data which will be made available to the @@ -4172,141 +4255,18 @@ paths: required: false schema: type: string - responses: - '200': - content: - application/jsonl: - schema: - oneOf: - - $ref: '#/components/schemas/VectorMemoryBank' - - $ref: '#/components/schemas/KeyValueMemoryBank' - - $ref: '#/components/schemas/KeywordMemoryBank' - - $ref: '#/components/schemas/GraphMemoryBank' - description: OK - tags: - - MemoryBanks - /alpha/memory-banks/register: - post: - parameters: - - description: JSON-encoded provider data which will be made available to the - adapter servicing the API - in: header - name: X-LlamaStack-Provider-Data - required: false - schema: - type: string - - description: Version of the client making the request. This is used to ensure - that the client and server are compatible. - in: header - name: X-LlamaStack-Client-Version - required: false - schema: - type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/RegisterMemoryBankRequest' - required: true - responses: {} - tags: - - MemoryBanks - /alpha/memory-banks/unregister: - post: - parameters: - - description: JSON-encoded provider data which will be made available to the - adapter servicing the API - in: header - name: X-LlamaStack-Provider-Data - required: false - schema: - type: string - - description: Version of the client making the request. This is used to ensure - that the client and server are compatible. - in: header - name: X-LlamaStack-Client-Version - required: false - schema: - type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/UnregisterMemoryBankRequest' - required: true - responses: - '200': - description: OK - tags: - - MemoryBanks - /alpha/memory/insert: - post: - parameters: - - description: JSON-encoded provider data which will be made available to the - adapter servicing the API - in: header - name: X-LlamaStack-Provider-Data - required: false - schema: - type: string - - description: Version of the client making the request. This is used to ensure - that the client and server are compatible. - in: header - name: X-LlamaStack-Client-Version - required: false - schema: - type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/InsertDocumentsRequest' - required: true - responses: - '200': - description: OK - tags: - - Memory - /alpha/memory/query: - post: - parameters: - - description: JSON-encoded provider data which will be made available to the - adapter servicing the API - in: header - name: X-LlamaStack-Provider-Data - required: false - schema: - type: string - - description: Version of the client making the request. This is used to ensure - that the client and server are compatible. - in: header - name: X-LlamaStack-Client-Version - required: false - schema: - type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/QueryDocumentsRequest' - required: true responses: '200': content: application/json: schema: - $ref: '#/components/schemas/QueryDocumentsResponse' + $ref: '#/components/schemas/ListProvidersResponse' description: OK tags: - - Memory - /alpha/models/get: + - Inspect + /v1/inspect/routes: get: parameters: - - in: query - name: identifier - required: true - schema: - type: string - description: JSON-encoded provider data which will be made available to the adapter servicing the API in: header @@ -4326,39 +4286,36 @@ paths: content: application/json: schema: - oneOf: - - $ref: '#/components/schemas/Model' - - type: 'null' + $ref: '#/components/schemas/ListRoutesResponse' + description: OK + tags: + - Inspect + /v1/models: + get: + parameters: + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-Provider-Data + required: false + schema: + type: string + - description: Version of the client making the request. This is used to ensure + that the client and server are compatible. + in: header + name: X-LlamaStack-Client-Version + required: false + schema: + type: string + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/ListModelsResponse' description: OK tags: - Models - /alpha/models/list: - get: - parameters: - - description: JSON-encoded provider data which will be made available to the - adapter servicing the API - in: header - name: X-LlamaStack-Provider-Data - required: false - schema: - type: string - - description: Version of the client making the request. This is used to ensure - that the client and server are compatible. - in: header - name: X-LlamaStack-Client-Version - required: false - schema: - type: string - responses: - '200': - content: - application/jsonl: - schema: - $ref: '#/components/schemas/Model' - description: OK - tags: - - Models - /alpha/models/register: post: parameters: - description: JSON-encoded provider data which will be made available to the @@ -4390,9 +4347,14 @@ paths: description: OK tags: - Models - /alpha/models/unregister: - post: + /v1/models/{model_id}: + delete: parameters: + - in: path + name: model_id + required: true + schema: + type: string - description: JSON-encoded provider data which will be made available to the adapter servicing the API in: header @@ -4407,18 +4369,44 @@ paths: required: false schema: type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/UnregisterModelRequest' - required: true responses: '200': description: OK tags: - Models - /alpha/post-training/job/artifacts: + get: + parameters: + - in: path + name: model_id + required: true + schema: + type: string + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-Provider-Data + required: false + schema: + type: string + - description: Version of the client making the request. This is used to ensure + that the client and server are compatible. + in: header + name: X-LlamaStack-Client-Version + required: false + schema: + type: string + responses: + '200': + content: + application/json: + schema: + oneOf: + - $ref: '#/components/schemas/Model' + - type: 'null' + description: OK + tags: + - Models + /v1/post-training/job/artifacts: get: parameters: - in: query @@ -4451,7 +4439,7 @@ paths: description: OK tags: - PostTraining (Coming Soon) - /alpha/post-training/job/cancel: + /v1/post-training/job/cancel: post: parameters: - description: JSON-encoded provider data which will be made available to the @@ -4479,7 +4467,7 @@ paths: description: OK tags: - PostTraining (Coming Soon) - /alpha/post-training/job/status: + /v1/post-training/job/status: get: parameters: - in: query @@ -4512,7 +4500,7 @@ paths: description: OK tags: - PostTraining (Coming Soon) - /alpha/post-training/jobs: + /v1/post-training/jobs: get: parameters: - description: JSON-encoded provider data which will be made available to the @@ -4532,13 +4520,13 @@ paths: responses: '200': content: - application/jsonl: + application/json: schema: - $ref: '#/components/schemas/PostTrainingJob' + $ref: '#/components/schemas/ListPostTrainingJobsResponse' description: OK tags: - PostTraining (Coming Soon) - /alpha/post-training/preference-optimize: + /v1/post-training/preference-optimize: post: parameters: - description: JSON-encoded provider data which will be made available to the @@ -4570,7 +4558,7 @@ paths: description: OK tags: - PostTraining (Coming Soon) - /alpha/post-training/supervised-fine-tune: + /v1/post-training/supervised-fine-tune: post: parameters: - description: JSON-encoded provider data which will be made available to the @@ -4602,65 +4590,7 @@ paths: description: OK tags: - PostTraining (Coming Soon) - /alpha/providers/list: - get: - parameters: - - description: JSON-encoded provider data which will be made available to the - adapter servicing the API - in: header - name: X-LlamaStack-Provider-Data - required: false - schema: - type: string - - description: Version of the client making the request. This is used to ensure - that the client and server are compatible. - in: header - name: X-LlamaStack-Client-Version - required: false - schema: - type: string - responses: - '200': - content: - application/json: - schema: - additionalProperties: - $ref: '#/components/schemas/ProviderInfo' - type: object - description: OK - tags: - - Inspect - /alpha/routes/list: - get: - parameters: - - description: JSON-encoded provider data which will be made available to the - adapter servicing the API - in: header - name: X-LlamaStack-Provider-Data - required: false - schema: - type: string - - description: Version of the client making the request. This is used to ensure - that the client and server are compatible. - in: header - name: X-LlamaStack-Client-Version - required: false - schema: - type: string - responses: - '200': - content: - application/json: - schema: - additionalProperties: - items: - $ref: '#/components/schemas/RouteInfo' - type: array - type: object - description: OK - tags: - - Inspect - /alpha/safety/run-shield: + /v1/safety/run-shield: post: parameters: - description: JSON-encoded provider data which will be made available to the @@ -4692,10 +4622,63 @@ paths: description: OK tags: - Safety - /alpha/scoring-functions/get: + /v1/scoring-functions: get: parameters: - - in: query + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-Provider-Data + required: false + schema: + type: string + - description: Version of the client making the request. This is used to ensure + that the client and server are compatible. + in: header + name: X-LlamaStack-Client-Version + required: false + schema: + type: string + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/ListScoringFunctionsResponse' + description: OK + tags: + - ScoringFunctions + post: + parameters: + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-Provider-Data + required: false + schema: + type: string + - description: Version of the client making the request. This is used to ensure + that the client and server are compatible. + in: header + name: X-LlamaStack-Client-Version + required: false + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/RegisterScoringFunctionRequest' + required: true + responses: + '200': + description: OK + tags: + - ScoringFunctions + /v1/scoring-functions/{scoring_fn_id}: + get: + parameters: + - in: path name: scoring_fn_id required: true schema: @@ -4725,61 +4708,7 @@ paths: description: OK tags: - ScoringFunctions - /alpha/scoring-functions/list: - get: - parameters: - - description: JSON-encoded provider data which will be made available to the - adapter servicing the API - in: header - name: X-LlamaStack-Provider-Data - required: false - schema: - type: string - - description: Version of the client making the request. This is used to ensure - that the client and server are compatible. - in: header - name: X-LlamaStack-Client-Version - required: false - schema: - type: string - responses: - '200': - content: - application/jsonl: - schema: - $ref: '#/components/schemas/ScoringFn' - description: OK - tags: - - ScoringFunctions - /alpha/scoring-functions/register: - post: - parameters: - - description: JSON-encoded provider data which will be made available to the - adapter servicing the API - in: header - name: X-LlamaStack-Provider-Data - required: false - schema: - type: string - - description: Version of the client making the request. This is used to ensure - that the client and server are compatible. - in: header - name: X-LlamaStack-Client-Version - required: false - schema: - type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/RegisterScoringFunctionRequest' - required: true - responses: - '200': - description: OK - tags: - - ScoringFunctions - /alpha/scoring/score: + /v1/scoring/score: post: parameters: - description: JSON-encoded provider data which will be made available to the @@ -4811,7 +4740,7 @@ paths: description: OK tags: - Scoring - /alpha/scoring/score-batch: + /v1/scoring/score-batch: post: parameters: - description: JSON-encoded provider data which will be made available to the @@ -4843,14 +4772,9 @@ paths: description: OK tags: - Scoring - /alpha/shields/get: + /v1/shields: get: parameters: - - in: query - name: identifier - required: true - schema: - type: string - description: JSON-encoded provider data which will be made available to the adapter servicing the API in: header @@ -4870,39 +4794,10 @@ paths: content: application/json: schema: - oneOf: - - $ref: '#/components/schemas/Shield' - - type: 'null' + $ref: '#/components/schemas/ListShieldsResponse' description: OK tags: - Shields - /alpha/shields/list: - get: - parameters: - - description: JSON-encoded provider data which will be made available to the - adapter servicing the API - in: header - name: X-LlamaStack-Provider-Data - required: false - schema: - type: string - - description: Version of the client making the request. This is used to ensure - that the client and server are compatible. - in: header - name: X-LlamaStack-Client-Version - required: false - schema: - type: string - responses: - '200': - content: - application/jsonl: - schema: - $ref: '#/components/schemas/Shield' - description: OK - tags: - - Shields - /alpha/shields/register: post: parameters: - description: JSON-encoded provider data which will be made available to the @@ -4934,7 +4829,40 @@ paths: description: OK tags: - Shields - /alpha/synthetic-data-generation/generate: + /v1/shields/{identifier}: + get: + parameters: + - in: path + name: identifier + required: true + schema: + type: string + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-Provider-Data + required: false + schema: + type: string + - description: Version of the client making the request. This is used to ensure + that the client and server are compatible. + in: header + name: X-LlamaStack-Client-Version + required: false + schema: + type: string + responses: + '200': + content: + application/json: + schema: + oneOf: + - $ref: '#/components/schemas/Shield' + - type: 'null' + description: OK + tags: + - Shields + /v1/synthetic-data-generation/generate: post: parameters: - description: JSON-encoded provider data which will be made available to the @@ -4966,51 +4894,7 @@ paths: description: OK tags: - SyntheticDataGeneration (Coming Soon) - /alpha/telemetry/get-span-tree: - post: - parameters: - - in: query - name: span_id - required: true - schema: - type: string - - in: query - name: max_depth - required: false - schema: - type: integer - - description: JSON-encoded provider data which will be made available to the - adapter servicing the API - in: header - name: X-LlamaStack-Provider-Data - required: false - schema: - type: string - - description: Version of the client making the request. This is used to ensure - that the client and server are compatible. - in: header - name: X-LlamaStack-Client-Version - required: false - schema: - type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/GetSpanTreeRequest' - required: true - responses: - '200': - content: - application/json: - schema: - additionalProperties: - $ref: '#/components/schemas/SpanWithStatus' - type: object - description: OK - tags: - - Telemetry - /alpha/telemetry/log-event: + /v1/telemetry/events: post: parameters: - description: JSON-encoded provider data which will be made available to the @@ -5038,9 +4922,28 @@ paths: description: OK tags: - Telemetry - /alpha/telemetry/query-spans: - post: + /v1/telemetry/spans: + get: parameters: + - in: query + name: attribute_filters + required: true + schema: + items: + $ref: '#/components/schemas/QueryCondition' + type: array + - in: query + name: attributes_to_return + required: true + schema: + items: + type: string + type: array + - in: query + name: max_depth + required: false + schema: + type: integer - description: JSON-encoded provider data which will be made available to the adapter servicing the API in: header @@ -5055,54 +4958,16 @@ paths: required: false schema: type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/QuerySpansRequest' - required: true responses: '200': content: - application/jsonl: + application/json: schema: - $ref: '#/components/schemas/Span' + $ref: '#/components/schemas/QuerySpansResponse' description: OK tags: - Telemetry - /alpha/telemetry/query-traces: - post: - parameters: - - description: JSON-encoded provider data which will be made available to the - adapter servicing the API - in: header - name: X-LlamaStack-Provider-Data - required: false - schema: - type: string - - description: Version of the client making the request. This is used to ensure - that the client and server are compatible. - in: header - name: X-LlamaStack-Client-Version - required: false - schema: - type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/QueryTracesRequest' - required: true - responses: - '200': - content: - application/jsonl: - schema: - $ref: '#/components/schemas/Trace' - description: OK - tags: - - Telemetry - /alpha/telemetry/save-spans-to-dataset: + /v1/telemetry/spans/export: post: parameters: - description: JSON-encoded provider data which will be made available to the @@ -5130,7 +4995,167 @@ paths: description: OK tags: - Telemetry - /alpha/tool-runtime/invoke: + /v1/telemetry/spans/{span_id}/tree: + get: + parameters: + - in: path + name: span_id + required: true + schema: + type: string + - in: query + name: attributes_to_return + required: false + schema: + items: + type: string + type: array + - in: query + name: max_depth + required: false + schema: + type: integer + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-Provider-Data + required: false + schema: + type: string + - description: Version of the client making the request. This is used to ensure + that the client and server are compatible. + in: header + name: X-LlamaStack-Client-Version + required: false + schema: + type: string + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/QuerySpanTreeResponse' + description: OK + tags: + - Telemetry + /v1/telemetry/traces: + get: + parameters: + - in: query + name: attribute_filters + required: false + schema: + items: + $ref: '#/components/schemas/QueryCondition' + type: array + - in: query + name: limit + required: false + schema: + type: integer + - in: query + name: offset + required: false + schema: + type: integer + - in: query + name: order_by + required: false + schema: + items: + type: string + type: array + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-Provider-Data + required: false + schema: + type: string + - description: Version of the client making the request. This is used to ensure + that the client and server are compatible. + in: header + name: X-LlamaStack-Client-Version + required: false + schema: + type: string + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/QueryTracesResponse' + description: OK + tags: + - Telemetry + /v1/telemetry/traces/{trace_id}: + get: + parameters: + - in: path + name: trace_id + required: true + schema: + type: string + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-Provider-Data + required: false + schema: + type: string + - description: Version of the client making the request. This is used to ensure + that the client and server are compatible. + in: header + name: X-LlamaStack-Client-Version + required: false + schema: + type: string + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/Trace' + description: OK + tags: + - Telemetry + /v1/telemetry/traces/{trace_id}/spans/{span_id}: + get: + parameters: + - in: path + name: trace_id + required: true + schema: + type: string + - in: path + name: span_id + required: true + schema: + type: string + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-Provider-Data + required: false + schema: + type: string + - description: Version of the client making the request. This is used to ensure + that the client and server are compatible. + in: header + name: X-LlamaStack-Client-Version + required: false + schema: + type: string + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/Span' + description: OK + tags: + - Telemetry + /v1/tool-runtime/invoke: post: parameters: - description: JSON-encoded provider data which will be made available to the @@ -5163,14 +5188,45 @@ paths: summary: Run a tool with the given arguments tags: - ToolRuntime - /alpha/tool-runtime/list-tools: - post: + /v1/tool-runtime/list-tools: + get: parameters: - in: query name: tool_group_id required: false schema: type: string + - in: query + name: mcp_endpoint + required: false + schema: + $ref: '#/components/schemas/URL' + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-Provider-Data + required: false + schema: + type: string + - description: Version of the client making the request. This is used to ensure + that the client and server are compatible. + in: header + name: X-LlamaStack-Client-Version + required: false + schema: + type: string + responses: + '200': + content: + application/jsonl: + schema: + $ref: '#/components/schemas/ToolDef' + description: OK + tags: + - ToolRuntime + /v1/tool-runtime/rag-tool/insert: + post: + parameters: - description: JSON-encoded provider data which will be made available to the adapter servicing the API in: header @@ -5189,25 +5245,50 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ListRuntimeToolsRequest' + $ref: '#/components/schemas/InsertRequest' + required: true + responses: + '200': + description: OK + summary: Index documents so they can be used by the RAG system + tags: + - ToolRuntime + /v1/tool-runtime/rag-tool/query: + post: + parameters: + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-Provider-Data + required: false + schema: + type: string + - description: Version of the client making the request. This is used to ensure + that the client and server are compatible. + in: header + name: X-LlamaStack-Client-Version + required: false + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/QueryRequest' required: true responses: '200': content: - application/jsonl: + application/json: schema: - $ref: '#/components/schemas/ToolDef' + $ref: '#/components/schemas/RAGQueryResult' description: OK + summary: Query the RAG system for context; typically invoked by the agent tags: - ToolRuntime - /alpha/toolgroups/get: + /v1/toolgroups: get: parameters: - - in: query - name: toolgroup_id - required: true - schema: - type: string - description: JSON-encoded provider data which will be made available to the adapter servicing the API in: header @@ -5227,38 +5308,11 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/ToolGroup' - description: OK - tags: - - ToolGroups - /alpha/toolgroups/list: - get: - parameters: - - description: JSON-encoded provider data which will be made available to the - adapter servicing the API - in: header - name: X-LlamaStack-Provider-Data - required: false - schema: - type: string - - description: Version of the client making the request. This is used to ensure - that the client and server are compatible. - in: header - name: X-LlamaStack-Client-Version - required: false - schema: - type: string - responses: - '200': - content: - application/jsonl: - schema: - $ref: '#/components/schemas/ToolGroup' + $ref: '#/components/schemas/ListToolGroupsResponse' description: OK summary: List tool groups with optional provider tags: - ToolGroups - /alpha/toolgroups/register: post: parameters: - description: JSON-encoded provider data which will be made available to the @@ -5287,9 +5341,14 @@ paths: summary: Register a tool group tags: - ToolGroups - /alpha/toolgroups/unregister: - post: + /v1/toolgroups/{toolgroup_id}: + delete: parameters: + - in: path + name: toolgroup_id + required: true + schema: + type: string - description: JSON-encoded provider data which will be made available to the adapter servicing the API in: header @@ -5304,22 +5363,78 @@ paths: required: false schema: type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/UnregisterToolGroupRequest' - required: true responses: '200': description: OK summary: Unregister a tool group tags: - ToolGroups - /alpha/tools/get: + get: + parameters: + - in: path + name: toolgroup_id + required: true + schema: + type: string + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-Provider-Data + required: false + schema: + type: string + - description: Version of the client making the request. This is used to ensure + that the client and server are compatible. + in: header + name: X-LlamaStack-Client-Version + required: false + schema: + type: string + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/ToolGroup' + description: OK + tags: + - ToolGroups + /v1/tools: get: parameters: - in: query + name: toolgroup_id + required: false + schema: + type: string + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-Provider-Data + required: false + schema: + type: string + - description: Version of the client making the request. This is used to ensure + that the client and server are compatible. + in: header + name: X-LlamaStack-Client-Version + required: false + schema: + type: string + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/ListToolsResponse' + description: OK + summary: List tools with optional tool group + tags: + - ToolGroups + /v1/tools/{tool_name}: + get: + parameters: + - in: path name: tool_name required: true schema: @@ -5347,14 +5462,97 @@ paths: description: OK tags: - ToolGroups - /alpha/tools/list: + /v1/vector-dbs: get: parameters: - - in: query - name: tool_group_id + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-Provider-Data required: false schema: type: string + - description: Version of the client making the request. This is used to ensure + that the client and server are compatible. + in: header + name: X-LlamaStack-Client-Version + required: false + schema: + type: string + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/ListVectorDBsResponse' + description: OK + tags: + - VectorDBs + post: + parameters: + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-Provider-Data + required: false + schema: + type: string + - description: Version of the client making the request. This is used to ensure + that the client and server are compatible. + in: header + name: X-LlamaStack-Client-Version + required: false + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/RegisterVectorDbRequest' + required: true + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/VectorDB' + description: OK + tags: + - VectorDBs + /v1/vector-dbs/{vector_db_id}: + delete: + parameters: + - in: path + name: vector_db_id + required: true + schema: + type: string + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-Provider-Data + required: false + schema: + type: string + - description: Version of the client making the request. This is used to ensure + that the client and server are compatible. + in: header + name: X-LlamaStack-Client-Version + required: false + schema: + type: string + responses: + '200': + description: OK + tags: + - VectorDBs + get: + parameters: + - in: path + name: vector_db_id + required: true + schema: + type: string - description: JSON-encoded provider data which will be made available to the adapter servicing the API in: header @@ -5372,14 +5570,75 @@ paths: responses: '200': content: - application/jsonl: + application/json: schema: - $ref: '#/components/schemas/Tool' + oneOf: + - $ref: '#/components/schemas/VectorDB' + - type: 'null' description: OK - summary: List tools with optional tool group tags: - - ToolGroups - /alpha/version: + - VectorDBs + /v1/vector-io/insert: + post: + parameters: + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-Provider-Data + required: false + schema: + type: string + - description: Version of the client making the request. This is used to ensure + that the client and server are compatible. + in: header + name: X-LlamaStack-Client-Version + required: false + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/InsertChunksRequest' + required: true + responses: + '200': + description: OK + tags: + - VectorIO + /v1/vector-io/query: + post: + parameters: + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-Provider-Data + required: false + schema: + type: string + - description: Version of the client making the request. This is used to ensure + that the client and server are compatible. + in: header + name: X-LlamaStack-Client-Version + required: false + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/QueryChunksRequest' + required: true + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/QueryChunksResponse' + description: OK + tags: + - VectorIO + /v1/version: get: parameters: - description: JSON-encoded provider data which will be made available to the @@ -5425,6 +5684,9 @@ tags: name: AgentStepResponse - description: name: AgentTool +- description: + name: AgentTurnInputType - description: 'Streamed agent execution response. @@ -5461,6 +5723,8 @@ tags: - description: name: AppendRowsRequest +- description: + name: ArrayType - description: name: BasicScoringFnParams @@ -5480,11 +5744,16 @@ tags: - description: name: BenchmarkEvalTaskConfig +- description: + name: BooleanType - description: name: BuiltinTool - description: name: CancelTrainingJobRequest +- description: + name: ChatCompletionInputType - description: name: ChatCompletionRequest @@ -5513,6 +5782,9 @@ tags: ' name: Checkpoint +- description: + name: CompletionInputType - description: name: CompletionMessage @@ -5548,14 +5820,13 @@ tags: name: DataConfig - description: name: Dataset +- description: + name: DatasetFormat - name: DatasetIO - name: Datasets -- description: - name: DeleteAgentsRequest -- description: - name: DeleteAgentsSessionRequest + name: DefaultRAGQueryGeneratorConfig - description: name: EfficiencyConfig @@ -5575,29 +5846,24 @@ tags: - description: name: EvaluateRowsRequest -- description: - name: GetAgentsSessionRequest -- description: - name: GetSpanTreeRequest -- description: - name: GraphMemoryBank -- description: - name: GraphMemoryBankParams + name: GreedySamplingStrategy - description: name: HealthInfo - description: name: ImageContentItem +- description: + name: ImageDelta - name: Inference - description: name: InferenceStep -- description: - name: InsertDocumentsRequest + name: InsertChunksRequest +- description: + name: InsertRequest - name: Inspect - description: @@ -5610,29 +5876,49 @@ tags: name: InvokeToolRequest - description: name: Job -- description: - name: JobCancelRequest - description: name: JobStatus -- description: - name: KeyValueMemoryBank -- description: - name: KeyValueMemoryBankParams -- description: - name: KeywordMemoryBank -- description: - name: KeywordMemoryBankParams +- description: + name: JsonType - description: name: LLMAsJudgeScoringFnParams -- description: - name: ListRuntimeToolsRequest + name: LLMRAGQueryGeneratorConfig +- description: + name: ListDatasetsResponse +- description: + name: ListEvalTasksResponse +- description: + name: ListModelsResponse +- description: + name: ListPostTrainingJobsResponse +- description: + name: ListProvidersResponse +- description: + name: ListRoutesResponse +- description: + name: ListScoringFunctionsResponse +- description: + name: ListShieldsResponse +- description: + name: ListToolGroupsResponse +- description: + name: ListToolsResponse +- description: + name: ListVectorDBsResponse - description: name: LogEventRequest @@ -5641,11 +5927,6 @@ tags: - description: name: LoraFinetuningConfig -- name: Memory -- description: - name: MemoryBankDocument -- name: MemoryBanks - description: name: MemoryRetrievalStep @@ -5660,6 +5941,10 @@ tags: - description: name: ModelType - name: Models +- description: + name: NumberType +- description: + name: ObjectType - description: name: OptimizerConfig @@ -5694,23 +5979,37 @@ tags: - description: name: QATFinetuningConfig +- description: + name: QueryChunksRequest +- description: + name: QueryChunksResponse - description: name: QueryCondition - description: name: QueryConditionOp -- description: + name: QueryRequest +- description: - name: QueryDocumentsRequest -- description: - name: QueryDocumentsResponse -- description: - name: QuerySpansRequest -- description: + name: RAGDocument +- description: + name: RAGQueryConfig +- description: - name: QueryTracesRequest + name: RAGQueryGeneratorConfig +- description: + name: RAGQueryResult - description: name: RegexParserScoringFnParams @@ -5720,9 +6019,6 @@ tags: - description: name: RegisterEvalTaskRequest -- description: - name: RegisterMemoryBankRequest - description: name: RegisterModelRequest @@ -5735,6 +6031,9 @@ tags: - description: name: RegisterToolGroupRequest +- description: + name: RegisterVectorDbRequest - description: name: ResponseFormat - description: @@ -5753,9 +6052,6 @@ tags: name: SafetyViolation - description: name: SamplingParams -- description: - name: SamplingStrategy - description: name: SaveSpansToDatasetRequest @@ -5801,6 +6097,8 @@ tags: name: SpanWithStatus - description: name: StopReason +- description: + name: StringType - description: name: StructuredLogEvent @@ -5824,6 +6122,8 @@ tags: - description: name: TextContentItem +- description: + name: TextDelta - description: name: TokenLogProbs - description: @@ -5874,6 +6174,12 @@ tags: /> name: ToolResponseMessage - name: ToolRuntime +- description: + name: TopKSamplingStrategy +- description: + name: TopPSamplingStrategy - description: name: Trace - description: @@ -5885,29 +6191,17 @@ tags: name: Turn - description: name: URL -- description: - name: UnregisterDatasetRequest -- description: - name: UnregisterMemoryBankRequest -- description: - name: UnregisterModelRequest -- description: - name: UnregisterToolGroupRequest +- description: + name: UnionType - description: name: UnstructuredLogEvent - description: name: UserMessage -- description: - name: VectorMemoryBank -- description: - name: VectorMemoryBankParams +- description: + name: VectorDB +- name: VectorDBs +- name: VectorIO - description: name: VersionInfo - description: @@ -5923,8 +6217,6 @@ x-tagGroups: - EvalTasks - Inference - Inspect - - Memory - - MemoryBanks - Models - PostTraining (Coming Soon) - Safety @@ -5935,6 +6227,8 @@ x-tagGroups: - Telemetry - ToolGroups - ToolRuntime + - VectorDBs + - VectorIO - name: Types tags: - AgentCandidate @@ -5943,6 +6237,7 @@ x-tagGroups: - AgentSessionCreateResponse - AgentStepResponse - AgentTool + - AgentTurnInputType - AgentTurnResponseEvent - AgentTurnResponseStepCompletePayload - AgentTurnResponseStepProgressPayload @@ -5953,20 +6248,24 @@ x-tagGroups: - AggregationFunctionType - AppEvalTaskConfig - AppendRowsRequest + - ArrayType - BasicScoringFnParams - BatchChatCompletionRequest - BatchChatCompletionResponse - BatchCompletionRequest - BatchCompletionResponse - BenchmarkEvalTaskConfig + - BooleanType - BuiltinTool - CancelTrainingJobRequest + - ChatCompletionInputType - ChatCompletionRequest - ChatCompletionResponse - ChatCompletionResponseEvent - ChatCompletionResponseEventType - ChatCompletionResponseStreamChunk - Checkpoint + - CompletionInputType - CompletionMessage - CompletionRequest - CompletionResponse @@ -5978,44 +6277,51 @@ x-tagGroups: - DPOAlignmentConfig - DataConfig - Dataset - - DeleteAgentsRequest - - DeleteAgentsSessionRequest + - DatasetFormat + - DefaultRAGQueryGeneratorConfig - EfficiencyConfig - EmbeddingsRequest - EmbeddingsResponse - EvalTask - EvaluateResponse - EvaluateRowsRequest - - GetAgentsSessionRequest - - GetSpanTreeRequest - - GraphMemoryBank - - GraphMemoryBankParams + - GreedySamplingStrategy - HealthInfo - ImageContentItem + - ImageDelta - InferenceStep - - InsertDocumentsRequest + - InsertChunksRequest + - InsertRequest - InterleavedContent - InterleavedContentItem - InvokeToolRequest - Job - - JobCancelRequest - JobStatus - - KeyValueMemoryBank - - KeyValueMemoryBankParams - - KeywordMemoryBank - - KeywordMemoryBankParams + - JsonType - LLMAsJudgeScoringFnParams - - ListRuntimeToolsRequest + - LLMRAGQueryGeneratorConfig + - ListDatasetsResponse + - ListEvalTasksResponse + - ListModelsResponse + - ListPostTrainingJobsResponse + - ListProvidersResponse + - ListRoutesResponse + - ListScoringFunctionsResponse + - ListShieldsResponse + - ListToolGroupsResponse + - ListToolsResponse + - ListVectorDBsResponse - LogEventRequest - LogSeverity - LoraFinetuningConfig - - MemoryBankDocument - MemoryRetrievalStep - Message - MetricEvent - Model - ModelCandidate - ModelType + - NumberType + - ObjectType - OptimizerConfig - OptimizerType - PaginatedRowsResult @@ -6026,20 +6332,26 @@ x-tagGroups: - PreferenceOptimizeRequest - ProviderInfo - QATFinetuningConfig + - QueryChunksRequest + - QueryChunksResponse - QueryCondition - QueryConditionOp - - QueryDocumentsRequest - - QueryDocumentsResponse - - QuerySpansRequest - - QueryTracesRequest + - QueryRequest + - QuerySpanTreeResponse + - QuerySpansResponse + - QueryTracesResponse + - RAGDocument + - RAGQueryConfig + - RAGQueryGeneratorConfig + - RAGQueryResult - RegexParserScoringFnParams - RegisterDatasetRequest - RegisterEvalTaskRequest - - RegisterMemoryBankRequest - RegisterModelRequest - RegisterScoringFunctionRequest - RegisterShieldRequest - RegisterToolGroupRequest + - RegisterVectorDbRequest - ResponseFormat - RouteInfo - RunEvalRequest @@ -6047,7 +6359,6 @@ x-tagGroups: - RunShieldResponse - SafetyViolation - SamplingParams - - SamplingStrategy - SaveSpansToDatasetRequest - ScoreBatchRequest - ScoreBatchResponse @@ -6064,12 +6375,14 @@ x-tagGroups: - SpanStatus - SpanWithStatus - StopReason + - StringType - StructuredLogEvent - SupervisedFineTuneRequest - SyntheticDataGenerateRequest - SyntheticDataGenerationResponse - SystemMessage - TextContentItem + - TextDelta - TokenLogProbs - Tool - ToolCall @@ -6087,17 +6400,15 @@ x-tagGroups: - ToolPromptFormat - ToolResponse - ToolResponseMessage + - TopKSamplingStrategy + - TopPSamplingStrategy - Trace - TrainingConfig - Turn - URL - - UnregisterDatasetRequest - - UnregisterMemoryBankRequest - - UnregisterModelRequest - - UnregisterToolGroupRequest + - UnionType - UnstructuredLogEvent - UserMessage - - VectorMemoryBank - - VectorMemoryBankParams + - VectorDB - VersionInfo - ViolationLevel diff --git a/docs/source/benchmark_evaluations/index.md b/docs/source/benchmark_evaluations/index.md index 240555936..56852c89c 100644 --- a/docs/source/benchmark_evaluations/index.md +++ b/docs/source/benchmark_evaluations/index.md @@ -56,9 +56,10 @@ response = client.eval.evaluate_rows( "type": "model", "model": "meta-llama/Llama-3.2-90B-Vision-Instruct", "sampling_params": { - "temperature": 0.0, + "strategy": { + "type": "greedy", + }, "max_tokens": 4096, - "top_p": 0.9, "repeat_penalty": 1.0, }, "system_message": system_message @@ -113,9 +114,10 @@ response = client.eval.evaluate_rows( "type": "model", "model": "meta-llama/Llama-3.2-90B-Vision-Instruct", "sampling_params": { - "temperature": 0.0, + "strategy": { + "type": "greedy", + }, "max_tokens": 4096, - "top_p": 0.9, "repeat_penalty": 1.0, }, } @@ -134,9 +136,9 @@ agent_config = { "model": "meta-llama/Llama-3.1-405B-Instruct", "instructions": "You are a helpful assistant", "sampling_params": { - "strategy": "greedy", - "temperature": 0.0, - "top_p": 0.95, + "strategy": { + "type": "greedy", + }, }, "tools": [ { diff --git a/docs/source/building_applications/agent_execution_loop.md b/docs/source/building_applications/agent_execution_loop.md new file mode 100644 index 000000000..62fb314bc --- /dev/null +++ b/docs/source/building_applications/agent_execution_loop.md @@ -0,0 +1,133 @@ +# Agent Execution Loop + +Agents are the heart of complex AI applications. They combine inference, memory, safety, and tool usage into coherent workflows. At its core, an agent follows a sophisticated execution loop that enables multi-step reasoning, tool usage, and safety checks. + +Each agent turn follows these key steps: + +1. **Initial Safety Check**: The user's input is first screened through configured safety shields + +2. **Context Retrieval**: + - If RAG is enabled, the agent queries relevant documents from memory banks + - For new documents, they are first inserted into the memory bank + - Retrieved context is augmented to the user's prompt + +3. **Inference Loop**: The agent enters its main execution loop: + - The LLM receives the augmented prompt (with context and/or previous tool outputs) + - The LLM generates a response, potentially with tool calls + - If tool calls are present: + - Tool inputs are safety-checked + - Tools are executed (e.g., web search, code execution) + - Tool responses are fed back to the LLM for synthesis + - The loop continues until: + - The LLM provides a final response without tool calls + - Maximum iterations are reached + - Token limit is exceeded + +4. **Final Safety Check**: The agent's final response is screened through safety shields + +```{mermaid} +sequenceDiagram + participant U as User + participant E as Executor + participant M as Memory Bank + participant L as LLM + participant T as Tools + participant S as Safety Shield + + Note over U,S: Agent Turn Start + U->>S: 1. Submit Prompt + activate S + S->>E: Input Safety Check + deactivate S + + E->>M: 2.1 Query Context + M-->>E: 2.2 Retrieved Documents + + loop Inference Loop + E->>L: 3.1 Augment with Context + L-->>E: 3.2 Response (with/without tool calls) + + alt Has Tool Calls + E->>S: Check Tool Input + S->>T: 4.1 Execute Tool + T-->>E: 4.2 Tool Response + E->>L: 5.1 Tool Response + L-->>E: 5.2 Synthesized Response + end + + opt Stop Conditions + Note over E: Break if: + Note over E: - No tool calls + Note over E: - Max iterations reached + Note over E: - Token limit exceeded + end + end + + E->>S: Output Safety Check + S->>U: 6. Final Response +``` + +Each step in this process can be monitored and controlled through configurations. Here's an example that demonstrates monitoring the agent's execution: + +```python +from llama_stack_client.lib.agents.event_logger import EventLogger + +agent_config = AgentConfig( + model="Llama3.2-3B-Instruct", + instructions="You are a helpful assistant", + # Enable both RAG and tool usage + tools=[ + { + "type": "memory", + "memory_bank_configs": [{ + "type": "vector", + "bank_id": "my_docs" + }], + "max_tokens_in_context": 4096 + }, + { + "type": "code_interpreter", + "enable_inline_code_execution": True + } + ], + # Configure safety + input_shields=["content_safety"], + output_shields=["content_safety"], + # Control the inference loop + max_infer_iters=5, + sampling_params={ + "strategy": { + "type": "top_p", + "temperature": 0.7, + "top_p": 0.95 + }, + "max_tokens": 2048 + } +) + +agent = Agent(client, agent_config) +session_id = agent.create_session("monitored_session") + +# Stream the agent's execution steps +response = agent.create_turn( + messages=[{"role": "user", "content": "Analyze this code and run it"}], + attachments=[{ + "content": "https://raw.githubusercontent.com/example/code.py", + "mime_type": "text/plain" + }], + session_id=session_id +) + +# Monitor each step of execution +for log in EventLogger().log(response): + if log.event.step_type == "memory_retrieval": + print("Retrieved context:", log.event.retrieved_context) + elif log.event.step_type == "inference": + print("LLM output:", log.event.model_response) + elif log.event.step_type == "tool_execution": + print("Tool call:", log.event.tool_call) + print("Tool response:", log.event.tool_response) + elif log.event.step_type == "shield_call": + if log.event.violation: + print("Safety violation:", log.event.violation) +``` diff --git a/docs/source/building_applications/evaluation.md b/docs/source/building_applications/evaluation.md new file mode 100644 index 000000000..473deaee2 --- /dev/null +++ b/docs/source/building_applications/evaluation.md @@ -0,0 +1,36 @@ +## Testing & Evaluation + +Llama Stack provides built-in tools for evaluating your applications: + +1. **Benchmarking**: Test against standard datasets +2. **Application Evaluation**: Score your application's outputs +3. **Custom Metrics**: Define your own evaluation criteria + +Here's how to set up basic evaluation: + +```python +# Create an evaluation task +response = client.eval_tasks.register( + eval_task_id="my_eval", + dataset_id="my_dataset", + scoring_functions=["accuracy", "relevance"] +) + +# Run evaluation +job = client.eval.run_eval( + task_id="my_eval", + task_config={ + "type": "app", + "eval_candidate": { + "type": "agent", + "config": agent_config + } + } +) + +# Get results +result = client.eval.job_result( + task_id="my_eval", + job_id=job.job_id +) +``` diff --git a/docs/source/building_applications/index.md b/docs/source/building_applications/index.md index acc19b515..6e1e9454f 100644 --- a/docs/source/building_applications/index.md +++ b/docs/source/building_applications/index.md @@ -1,421 +1,26 @@ # Building AI Applications -[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1F2ksmkoGQPa4pzRjMOE6BXWeOxWFIW6n?usp=sharing) +Llama Stack provides all the building blocks needed to create sophisticated AI applications. -Llama Stack provides all the building blocks needed to create sophisticated AI applications. This guide will walk you through how to use these components effectively. Check out our Colab notebook on to follow along working examples on how you can build LLM-powered agentic applications using Llama Stack. +The best way to get started is to look at this notebook which walks through the various APIs (from basic inference, to RAG agents) and how to use them. -## Basic Inference +**Notebook**: [Building AI Applications](docs/notebooks/Llama_Stack_Building_AI_Applications.ipynb) -The foundation of any AI application is the ability to interact with LLM models. Llama Stack provides a simple interface for both completion and chat-based inference: +## Agentic Concepts +- **[Agent Execution Loop](agent_execution_loop)** +- **[RAG](rag)** +- **[Safety](safety)** +- **[Tools](tools)** +- **[Telemetry](telemetry)** -```python -from llama_stack_client import LlamaStackClient - -client = LlamaStackClient(base_url="http://localhost:5001") - -# List available models -models = client.models.list() - -# Simple chat completion -response = client.inference.chat_completion( - model_id="Llama3.2-3B-Instruct", - messages=[ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Write a haiku about coding"} - ] -) -print(response.completion_message.content) -``` - -## Adding Memory & RAG - -Memory enables your applications to reference and recall information from previous interactions or external documents. Llama Stack's memory system is built around the concept of Memory Banks: - -1. **Vector Memory Banks**: For semantic search and retrieval -2. **Key-Value Memory Banks**: For structured data storage -3. **Keyword Memory Banks**: For basic text search -4. **Graph Memory Banks**: For relationship-based retrieval - -Here's how to set up a vector memory bank for RAG: - -```python -# Register a memory bank -bank_id = "my_documents" -response = client.memory_banks.register( - memory_bank_id=bank_id, - params={ - "memory_bank_type": "vector", - "embedding_model": "all-MiniLM-L6-v2", - "chunk_size_in_tokens": 512 - } -) - -# Insert documents -documents = [ - { - "document_id": "doc1", - "content": "Your document text here", - "mime_type": "text/plain" - } -] -client.memory.insert(bank_id, documents) - -# Query documents -results = client.memory.query( - bank_id=bank_id, - query="What do you know about...", -) -``` - -## Implementing Safety Guardrails - -Safety is a critical component of any AI application. Llama Stack provides a Shield system that can be applied at multiple touchpoints: - -```python -# Register a safety shield -shield_id = "content_safety" -client.shields.register( - shield_id=shield_id, - provider_shield_id="llama-guard-basic" -) - -# Run content through shield -response = client.safety.run_shield( - shield_id=shield_id, - messages=[{"role": "user", "content": "User message here"}] -) - -if response.violation: - print(f"Safety violation detected: {response.violation.user_message}") -``` - -## Building Agents - -Agents are the heart of complex AI applications. They combine inference, memory, safety, and tool usage into coherent workflows. At its core, an agent follows a sophisticated execution loop that enables multi-step reasoning, tool usage, and safety checks. - -### The Agent Execution Loop - -Each agent turn follows these key steps: - -1. **Initial Safety Check**: The user's input is first screened through configured safety shields - -2. **Context Retrieval**: - - If RAG is enabled, the agent queries relevant documents from memory banks - - For new documents, they are first inserted into the memory bank - - Retrieved context is augmented to the user's prompt - -3. **Inference Loop**: The agent enters its main execution loop: - - The LLM receives the augmented prompt (with context and/or previous tool outputs) - - The LLM generates a response, potentially with tool calls - - If tool calls are present: - - Tool inputs are safety-checked - - Tools are executed (e.g., web search, code execution) - - Tool responses are fed back to the LLM for synthesis - - The loop continues until: - - The LLM provides a final response without tool calls - - Maximum iterations are reached - - Token limit is exceeded - -4. **Final Safety Check**: The agent's final response is screened through safety shields - -```{mermaid} -sequenceDiagram - participant U as User - participant E as Executor - participant M as Memory Bank - participant L as LLM - participant T as Tools - participant S as Safety Shield - - Note over U,S: Agent Turn Start - U->>S: 1. Submit Prompt - activate S - S->>E: Input Safety Check - deactivate S - - E->>M: 2.1 Query Context - M-->>E: 2.2 Retrieved Documents - - loop Inference Loop - E->>L: 3.1 Augment with Context - L-->>E: 3.2 Response (with/without tool calls) - - alt Has Tool Calls - E->>S: Check Tool Input - S->>T: 4.1 Execute Tool - T-->>E: 4.2 Tool Response - E->>L: 5.1 Tool Response - L-->>E: 5.2 Synthesized Response - end - - opt Stop Conditions - Note over E: Break if: - Note over E: - No tool calls - Note over E: - Max iterations reached - Note over E: - Token limit exceeded - end - end - - E->>S: Output Safety Check - S->>U: 6. Final Response -``` - -Each step in this process can be monitored and controlled through configurations. Here's an example that demonstrates monitoring the agent's execution: - -```python -from llama_stack_client.lib.agents.event_logger import EventLogger - -agent_config = AgentConfig( - model="Llama3.2-3B-Instruct", - instructions="You are a helpful assistant", - # Enable both RAG and tool usage - tools=[ - { - "type": "memory", - "memory_bank_configs": [{ - "type": "vector", - "bank_id": "my_docs" - }], - "max_tokens_in_context": 4096 - }, - { - "type": "code_interpreter", - "enable_inline_code_execution": True - } - ], - # Configure safety - input_shields=["content_safety"], - output_shields=["content_safety"], - # Control the inference loop - max_infer_iters=5, - sampling_params={ - "temperature": 0.7, - "max_tokens": 2048 - } -) - -agent = Agent(client, agent_config) -session_id = agent.create_session("monitored_session") - -# Stream the agent's execution steps -response = agent.create_turn( - messages=[{"role": "user", "content": "Analyze this code and run it"}], - attachments=[{ - "content": "https://raw.githubusercontent.com/example/code.py", - "mime_type": "text/plain" - }], - session_id=session_id -) - -# Monitor each step of execution -for log in EventLogger().log(response): - if log.event.step_type == "memory_retrieval": - print("Retrieved context:", log.event.retrieved_context) - elif log.event.step_type == "inference": - print("LLM output:", log.event.model_response) - elif log.event.step_type == "tool_execution": - print("Tool call:", log.event.tool_call) - print("Tool response:", log.event.tool_response) - elif log.event.step_type == "shield_call": - if log.event.violation: - print("Safety violation:", log.event.violation) -``` - -This example shows how an agent can: Llama Stack provides a high-level agent framework: - -```python -from llama_stack_client.lib.agents.agent import Agent -from llama_stack_client.types.agent_create_params import AgentConfig - -# Configure an agent -agent_config = AgentConfig( - model="Llama3.2-3B-Instruct", - instructions="You are a helpful assistant", - tools=[ - { - "type": "memory", - "memory_bank_configs": [], - "query_generator_config": { - "type": "default", - "sep": " " - } - } - ], - input_shields=["content_safety"], - output_shields=["content_safety"], - enable_session_persistence=True -) - -# Create an agent -agent = Agent(client, agent_config) -session_id = agent.create_session("my_session") - -# Run agent turns -response = agent.create_turn( - messages=[{"role": "user", "content": "Your question here"}], - session_id=session_id -) -``` - -### Adding Tools to Agents - -Agents can be enhanced with various tools: - -1. **Search**: Web search capabilities through providers like Brave -2. **Code Interpreter**: Execute code snippets -3. **RAG**: Memory and document retrieval -4. **Function Calling**: Custom function execution -5. **WolframAlpha**: Mathematical computations -6. **Photogen**: Image generation - -Example of configuring an agent with tools: - -```python -agent_config = AgentConfig( - model="Llama3.2-3B-Instruct", - tools=[ - { - "type": "brave_search", - "api_key": "YOUR_API_KEY", - "engine": "brave" - }, - { - "type": "code_interpreter", - "enable_inline_code_execution": True - } - ], - tool_choice="auto", - tool_prompt_format="json" -) -``` - -## Building RAG-Enhanced Agents - -One of the most powerful patterns is combining agents with RAG capabilities. Here's a complete example: - -```python -from llama_stack_client.types import Attachment - -# Create attachments from documents -attachments = [ - Attachment( - content="https://raw.githubusercontent.com/example/doc.rst", - mime_type="text/plain" - ) -] - -# Configure agent with memory -agent_config = AgentConfig( - model="Llama3.2-3B-Instruct", - instructions="You are a helpful assistant", - tools=[{ - "type": "memory", - "memory_bank_configs": [], - "query_generator_config": {"type": "default", "sep": " "}, - "max_tokens_in_context": 4096, - "max_chunks": 10 - }], - enable_session_persistence=True -) - -agent = Agent(client, agent_config) -session_id = agent.create_session("rag_session") - -# Initial document ingestion -response = agent.create_turn( - messages=[{ - "role": "user", - "content": "I am providing some documents for reference." - }], - attachments=attachments, - session_id=session_id -) - -# Query with RAG -response = agent.create_turn( - messages=[{ - "role": "user", - "content": "What are the key topics in the documents?" - }], - session_id=session_id -) -``` - -## Testing & Evaluation - -Llama Stack provides built-in tools for evaluating your applications: - -1. **Benchmarking**: Test against standard datasets -2. **Application Evaluation**: Score your application's outputs -3. **Custom Metrics**: Define your own evaluation criteria - -Here's how to set up basic evaluation: - -```python -# Create an evaluation task -response = client.eval_tasks.register( - eval_task_id="my_eval", - dataset_id="my_dataset", - scoring_functions=["accuracy", "relevance"] -) - -# Run evaluation -job = client.eval.run_eval( - task_id="my_eval", - task_config={ - "type": "app", - "eval_candidate": { - "type": "agent", - "config": agent_config - } - } -) - -# Get results -result = client.eval.job_result( - task_id="my_eval", - job_id=job.job_id -) -``` - -## Debugging & Monitoring - -Llama Stack includes comprehensive telemetry for debugging and monitoring your applications: - -1. **Tracing**: Track request flows across components -2. **Metrics**: Measure performance and usage -3. **Logging**: Debug issues and track behavior - -The telemetry system supports multiple output formats: - -- OpenTelemetry for visualization in tools like Jaeger -- SQLite for local storage and querying -- Console output for development - -Example of querying traces: - -```python -# Query traces for a session -traces = client.telemetry.query_traces( - attribute_filters=[{ - "key": "session_id", - "op": "eq", - "value": session_id - }] -) - -# Get spans within the root span; indexed by ID -# Use parent_span_id to build a tree out of it -spans_by_id = client.telemetry.get_span_tree( - span_id=traces[0].root_span_id -) -``` - -For details on how to use the telemetry system to debug your applications, export traces to a dataset, and run evaluations, see the [Telemetry](telemetry) section. ```{toctree} :hidden: -:maxdepth: 3 +:maxdepth: 1 +agent_execution_loop +rag +safety +tools telemetry ``` diff --git a/docs/source/building_applications/rag.md b/docs/source/building_applications/rag.md new file mode 100644 index 000000000..17ecd2046 --- /dev/null +++ b/docs/source/building_applications/rag.md @@ -0,0 +1,92 @@ +## Memory & RAG + +Memory enables your applications to reference and recall information from previous interactions or external documents. Llama Stack's memory system is built around the concept of Memory Banks: + +1. **Vector Memory Banks**: For semantic search and retrieval +2. **Key-Value Memory Banks**: For structured data storage +3. **Keyword Memory Banks**: For basic text search +4. **Graph Memory Banks**: For relationship-based retrieval + +Here's how to set up a vector memory bank for RAG: + +```python +# Register a memory bank +bank_id = "my_documents" +response = client.memory_banks.register( + memory_bank_id=bank_id, + params={ + "memory_bank_type": "vector", + "embedding_model": "all-MiniLM-L6-v2", + "chunk_size_in_tokens": 512 + } +) + +# Insert documents +documents = [ + { + "document_id": "doc1", + "content": "Your document text here", + "mime_type": "text/plain" + } +] +client.memory.insert(bank_id, documents) + +# Query documents +results = client.memory.query( + bank_id=bank_id, + query="What do you know about...", +) +``` + + +### Building RAG-Enhanced Agents + +One of the most powerful patterns is combining agents with RAG capabilities. Here's a complete example: + +```python +from llama_stack_client.types import Attachment + +# Create attachments from documents +attachments = [ + Attachment( + content="https://raw.githubusercontent.com/example/doc.rst", + mime_type="text/plain" + ) +] + +# Configure agent with memory +agent_config = AgentConfig( + model="Llama3.2-3B-Instruct", + instructions="You are a helpful assistant", + tools=[{ + "type": "memory", + "memory_bank_configs": [], + "query_generator_config": {"type": "default", "sep": " "}, + "max_tokens_in_context": 4096, + "max_chunks": 10 + }], + enable_session_persistence=True +) + +agent = Agent(client, agent_config) +session_id = agent.create_session("rag_session") + +# Initial document ingestion +response = agent.create_turn( + messages=[{ + "role": "user", + "content": "I am providing some documents for reference." + }], + attachments=attachments, + session_id=session_id +) + +# Query with RAG +response = agent.create_turn( + messages=[{ + "role": "user", + "content": "What are the key topics in the documents?" + }], + session_id=session_id +) +``` diff --git a/docs/source/building_applications/safety.md b/docs/source/building_applications/safety.md new file mode 100644 index 000000000..31efa0f8c --- /dev/null +++ b/docs/source/building_applications/safety.md @@ -0,0 +1,21 @@ +## Safety Guardrails + +Safety is a critical component of any AI application. Llama Stack provides a Shield system that can be applied at multiple touchpoints: + +```python +# Register a safety shield +shield_id = "content_safety" +client.shields.register( + shield_id=shield_id, + provider_shield_id="llama-guard-basic" +) + +# Run content through shield +response = client.safety.run_shield( + shield_id=shield_id, + messages=[{"role": "user", "content": "User message here"}] +) + +if response.violation: + print(f"Safety violation detected: {response.violation.user_message}") +``` diff --git a/docs/source/building_applications/telemetry.md b/docs/source/building_applications/telemetry.md index 6c8067035..45bc7a1c2 100644 --- a/docs/source/building_applications/telemetry.md +++ b/docs/source/building_applications/telemetry.md @@ -1,8 +1,4 @@ # Telemetry -```{note} -The telemetry system is currently experimental and subject to change. We welcome feedback and contributions to help improve it. -``` - The Llama Stack telemetry system provides comprehensive tracing, metrics, and logging capabilities. It supports multiple sink types including OpenTelemetry, SQLite, and Console output. @@ -44,58 +40,6 @@ structured_log_event = SpanStartPayload( - **SQLite**: Store events in a local SQLite database. This is needed if you want to query the events later through the Llama Stack API. - **Console**: Print events to the console. -## APIs - -The telemetry API is designed to be flexible for different user flows like debugging/visualization in UI, monitoring, and saving traces to datasets. -The telemetry system exposes the following HTTP endpoints: - -### Log Event -```http -POST /telemetry/log-event -``` -Logs a telemetry event (unstructured log, metric, or structured log) with optional TTL. - -### Query Traces -```http -POST /telemetry/query-traces -``` -Retrieves traces based on filters with pagination support. Parameters: -- `attribute_filters`: List of conditions to filter traces -- `limit`: Maximum number of traces to return (default: 100) -- `offset`: Number of traces to skip (default: 0) -- `order_by`: List of fields to sort by - -### Get Span Tree -```http -POST /telemetry/get-span-tree -``` -Retrieves a hierarchical view of spans starting from a specific span. Parameters: -- `span_id`: ID of the root span to retrieve -- `attributes_to_return`: Optional list of specific attributes to include -- `max_depth`: Optional maximum depth of the span tree to return - -### Query Spans -```http -POST /telemetry/query-spans -``` -Retrieves spans matching specified filters and returns selected attributes. Parameters: -- `attribute_filters`: List of conditions to filter traces -- `attributes_to_return`: List of specific attributes to include in results -- `max_depth`: Optional maximum depth of spans to traverse (default: no limit) - -Returns a flattened list of spans with requested attributes. - -### Save Spans to Dataset -This is useful for saving traces to a dataset for running evaluations. For example, you can save the input/output of each span that is part of an agent session/turn to a dataset and then run an eval task on it. See example in [Example: Save Spans to Dataset](#example-save-spans-to-dataset). -```http -POST /telemetry/save-spans-to-dataset -``` -Queries spans and saves their attributes to a dataset. Parameters: -- `attribute_filters`: List of conditions to filter traces -- `attributes_to_save`: List of span attributes to save to the dataset -- `dataset_id`: ID of the dataset to save to -- `max_depth`: Optional maximum depth of spans to traverse (default: no limit) - ## Providers ### Meta-Reference Provider @@ -133,110 +77,4 @@ Once the Jaeger instance is running, you can visualize traces by navigating to h ## Querying Traces Stored in SQLIte -The `sqlite` sink allows you to query traces without an external system. Here are some example queries: - -Querying Traces for a agent session -The client SDK is not updated to support the new telemetry API. It will be updated soon. You can manually query traces using the following curl command: - -``` bash - curl -X POST 'http://localhost:5000/alpha/telemetry/query-traces' \ --H 'Content-Type: application/json' \ --d '{ - "attribute_filters": [ - { - "key": "session_id", - "op": "eq", - "value": "dd667b87-ca4b-4d30-9265-5a0de318fc65" }], - "limit": 100, - "offset": 0, - "order_by": ["start_time"] - - [ - { - "trace_id": "6902f54b83b4b48be18a6f422b13e16f", - "root_span_id": "5f37b85543afc15a", - "start_time": "2024-12-04T08:08:30.501587", - "end_time": "2024-12-04T08:08:36.026463" - }, - ........ -] -}' - -``` - -Querying spans for a specifc root span id - -``` bash -curl -X POST 'http://localhost:5000/alpha/telemetry/get-span-tree' \ --H 'Content-Type: application/json' \ --d '{ "span_id" : "6cceb4b48a156913", "max_depth": 2 }' - -{ - "span_id": "6cceb4b48a156913", - "trace_id": "dafa796f6aaf925f511c04cd7c67fdda", - "parent_span_id": "892a66d726c7f990", - "name": "retrieve_rag_context", - "start_time": "2024-12-04T09:28:21.781995", - "end_time": "2024-12-04T09:28:21.913352", - "attributes": { - "input": [ - "{\"role\":\"system\",\"content\":\"You are a helpful assistant\"}", - "{\"role\":\"user\",\"content\":\"What are the top 5 topics that were explained in the documentation? Only list succinct bullet points.\",\"context\":null}" - ] - }, - "children": [ - { - "span_id": "1a2df181854064a8", - "trace_id": "dafa796f6aaf925f511c04cd7c67fdda", - "parent_span_id": "6cceb4b48a156913", - "name": "MemoryRouter.query_documents", - "start_time": "2024-12-04T09:28:21.787620", - "end_time": "2024-12-04T09:28:21.906512", - "attributes": { - "input": null - }, - "children": [], - "status": "ok" - } - ], - "status": "ok" -} - -``` - -## Example: Save Spans to Dataset -Save all spans for a specific agent session to a dataset. -``` bash -curl -X POST 'http://localhost:5000/alpha/telemetry/save-spans-to-dataset' \ --H 'Content-Type: application/json' \ --d '{ - "attribute_filters": [ - { - "key": "session_id", - "op": "eq", - "value": "dd667b87-ca4b-4d30-9265-5a0de318fc65" - } - ], - "attributes_to_save": ["input", "output"], - "dataset_id": "my_dataset", - "max_depth": 10 -}' -``` - -Save all spans for a specific agent turn to a dataset. -```bash -curl -X POST 'http://localhost:5000/alpha/telemetry/save-spans-to-dataset' \ --H 'Content-Type: application/json' \ --d '{ - "attribute_filters": [ - { - "key": "turn_id", - "op": "eq", - "value": "123e4567-e89b-12d3-a456-426614174000" - } - ], - "attributes_to_save": ["input", "output"], - "dataset_id": "my_dataset", - "max_depth": 10 -}' -``` +The `sqlite` sink allows you to query traces without an external system. Here are some example queries. Refer to the notebook at [Llama Stack Building AI Applications](https://github.com/meta-llama/llama-stack/blob/main/docs/getting_started.ipynb) for more examples on how to query traces and spaces. diff --git a/docs/source/building_applications/tools.md b/docs/source/building_applications/tools.md new file mode 100644 index 000000000..81b4ab68e --- /dev/null +++ b/docs/source/building_applications/tools.md @@ -0,0 +1,202 @@ +# Tools + +Tools are functions that can be invoked by an agent to perform tasks. They are organized into tool groups and registered with specific providers. Each tool group represents a collection of related tools from a single provider. They are organized into groups so that state can be externalized: the collection operates on the same state typically. +An example of this would be a "db_access" tool group that contains tools for interacting with a database. "list_tables", "query_table", "insert_row" could be examples of tools in this group. + +Tools are treated as any other resource in llama stack like models. You can register them, have providers for them etc. + +When instatiating an agent, you can provide it a list of tool groups that it has access to. Agent gets the corresponding tool definitions for the specified tool groups and passes them along to the model. + +Refer to the [Building AI Applications](https://github.com/meta-llama/llama-stack/blob/main/docs/getting_started.ipynb) notebook for more examples on how to use tools. + +## Types of Tool Group providers + +There are three types of providers for tool groups that are supported by Llama Stack. + +1. Built-in providers +2. Model Context Protocol (MCP) providers +3. Client provided tools + +### Built-in providers + +Built-in providers come packaged with Llama Stack. These providers provide common functionalities like web search, code interpretation, and computational capabilities. + +#### Web Search providers +There are three web search providers that are supported by Llama Stack. + +1. Brave Search +2. Bing Search +3. Tavily Search + +Example client SDK call to register a "websearch" toolgroup that is provided by brave-search. + +```python +# Register Brave Search tool group +client.toolgroups.register( + toolgroup_id="builtin::websearch", + provider_id="brave-search", + args={"max_results": 5} +) +``` + +The tool requires an API key which can be provided either in the configuration or through the request header `X-LlamaStack-Provider-Data`. The format of the header is `{"_api_key": }`. + + + +#### Code Interpreter + +The Code Interpreter allows execution of Python code within a controlled environment. + +```python +# Register Code Interpreter tool group +client.toolgroups.register( + toolgroup_id="builtin::code_interpreter", + provider_id="code_interpreter" +) +``` + +Features: +- Secure execution environment using `bwrap` sandboxing +- Matplotlib support for generating plots +- Disabled dangerous system operations +- Configurable execution timeouts + +#### WolframAlpha + +The WolframAlpha tool provides access to computational knowledge through the WolframAlpha API. + +```python +# Register WolframAlpha tool group +client.toolgroups.register( + toolgroup_id="builtin::wolfram_alpha", + provider_id="wolfram-alpha" +) +``` + +Example usage: +```python +result = client.tool_runtime.invoke_tool( + tool_name="wolfram_alpha", + args={"query": "solve x^2 + 2x + 1 = 0"} +) +``` + +#### Memory + +The Memory tool enables retrieval of context from various types of memory banks (vector, key-value, keyword, and graph). + +```python +# Register Memory tool group +client.toolgroups.register( + toolgroup_id="builtin::memory", + provider_id="memory", + args={ + "max_chunks": 5, + "max_tokens_in_context": 4096 + } +) +``` + +Features: +- Support for multiple memory bank types +- Configurable query generation +- Context retrieval with token limits + + +> **Note:** By default, llama stack run.yaml defines toolgroups for web search, code interpreter and memory, that are provided by tavily-search, code-interpreter and memory providers. + +## Model Context Protocol (MCP) Tools + +MCP tools are special tools that can interact with llama stack over model context protocol. These tools are dynamically discovered from an MCP endpoint and can be used to extend the agent's capabilities. + +Refer to https://github.com/modelcontextprotocol/server for available MCP servers. + +```python +# Register MCP tools +client.toolgroups.register( + toolgroup_id="builtin::filesystem", + provider_id="model-context-protocol", + mcp_endpoint=URL(uri="http://localhost:8000/sse"), +) +``` + +MCP tools require: +- A valid MCP endpoint URL +- The endpoint must implement the Model Context Protocol +- Tools are discovered dynamically from the endpoint + + +## Tools provided by the client + +These tools are registered along with the agent config and are specific to the agent for which they are registered. The main difference between these tools and the tools provided by the built-in providers is that the execution of these tools is handled by the client and the agent transfers the tool call to the client and waits for the result from the client. + +```python +# Example agent config with client provided tools +config = AgentConfig( + toolgroups=[ + "builtin::websearch", + ], + client_tools=[ + ToolDef(name="client_tool", description="Client provided tool") + ] +) +``` + +Refer to [llama-stack-apps](https://github.com/meta-llama/llama-stack-apps/blob/main/examples/agents/e2e_loop_with_custom_tools.py) for an example of how to use client provided tools. + +## Tool Structure + +Each tool has the following components: + +- `name`: Unique identifier for the tool +- `description`: Human-readable description of the tool's functionality +- `parameters`: List of parameters the tool accepts + - `name`: Parameter name + - `parameter_type`: Data type (string, number, etc.) + - `description`: Parameter description + - `required`: Whether the parameter is required (default: true) + - `default`: Default value if any + +Example tool definition: +```python +{ + "name": "web_search", + "description": "Search the web for information", + "parameters": [ + { + "name": "query", + "parameter_type": "string", + "description": "The query to search for", + "required": True + } + ] +} +``` + +## Tool Invocation + +Tools can be invoked using the `invoke_tool` method: + +```python +result = client.tool_runtime.invoke_tool( + tool_name="web_search", + kwargs={"query": "What is the capital of France?"} +) +``` + +The result contains: +- `content`: The tool's output +- `error_message`: Optional error message if the tool failed +- `error_code`: Optional error code if the tool failed + +## Listing Available Tools + +You can list all available tools or filter by tool group: + +```python +# List all tools +all_tools = client.tools.list_tools() + +# List tools in a specific group +group_tools = client.tools.list_tools(toolgroup_id="search_tools") +``` diff --git a/docs/source/concepts/index.md b/docs/source/concepts/index.md index 5545c3103..f638ba8d0 100644 --- a/docs/source/concepts/index.md +++ b/docs/source/concepts/index.md @@ -10,7 +10,6 @@ A Llama Stack API is described as a collection of REST endpoints. We currently s - **Inference**: run inference with a LLM - **Safety**: apply safety policies to the output at a Systems (not only model) level - **Agents**: run multi-step agentic workflows with LLMs with tool usage, memory (RAG), etc. -- **Memory**: store and retrieve data for RAG, chat history, etc. - **DatasetIO**: interface with datasets and data loaders - **Scoring**: evaluate outputs of the system - **Eval**: generate outputs (via Inference or Agents) and perform scoring @@ -39,7 +38,6 @@ Some of these APIs are associated with a set of **Resources**. Here is the mappi - **Inference**, **Eval** and **Post Training** are associated with `Model` resources. - **Safety** is associated with `Shield` resources. -- **Memory** is associated with `Memory Bank` resources. - **DatasetIO** is associated with `Dataset` resources. - **Scoring** is associated with `ScoringFunction` resources. - **Eval** is associated with `Model` and `EvalTask` resources. @@ -63,12 +61,9 @@ While there is a lot of flexibility to mix-and-match providers, often users will **On-device Distro**: Finally, you may want to run Llama Stack directly on an edge device (mobile phone or a tablet.) We provide Distros for iOS and Android (coming soon.) -## More Concepts -- [Evaluation Concepts](evaluation_concepts.md) - ```{toctree} :maxdepth: 1 :hidden: -evaluation_concepts +distributions/index ``` diff --git a/docs/source/contributing/new_api_provider.md b/docs/source/contributing/new_api_provider.md index 3fa875c50..439021685 100644 --- a/docs/source/contributing/new_api_provider.md +++ b/docs/source/contributing/new_api_provider.md @@ -1,26 +1,57 @@ # Adding a New API Provider -This guide contains references to walk you through adding a new API provider. +This guide will walk you through the process of adding a new API provider to Llama Stack. -1. First, decide which API your provider falls into (e.g. Inference, Safety, Agents, Memory). -2. Decide whether your provider is a remote provider, or inline implementation. A remote provider is a provider that makes a remote request to a service. An inline provider is a provider where implementation is executed locally. Checkout the examples, and follow the structure to add your own API provider. Please find the following code pointers: +## Getting Started - - {repopath}`Remote Providers::llama_stack/providers/remote` - - {repopath}`Inline Providers::llama_stack/providers/inline` +1. **Choose Your API Category** + - Determine which API category your provider belongs to (Inference, Safety, Agents, VectorIO) + - Review the core concepts of Llama Stack in the [concepts guide](../concepts/index.md) -3. [Build a Llama Stack distribution](https://llama-stack.readthedocs.io/en/latest/distributions/building_distro.html) with your API provider. -4. Test your code! +2. **Determine Provider Type** + - **Remote Provider**: Makes requests to external services + - **Inline Provider**: Executes implementation locally -## Testing your newly added API providers + Reference existing implementations: + - {repopath}`Remote Providers::llama_stack/providers/remote` + - {repopath}`Inline Providers::llama_stack/providers/inline` -1. Start with an _integration test_ for your provider. That means we will instantiate the real provider, pass it real configuration and if it is a remote service, we will actually hit the remote service. We **strongly** discourage mocking for these tests at the provider level. Llama Stack is first and foremost about integration so we need to make sure stuff works end-to-end. See {repopath}`llama_stack/providers/tests/inference/test_text_inference.py` for an example. + Example PRs: + - [Grok Inference Implementation](https://github.com/meta-llama/llama-stack/pull/609) + - [Nvidia Inference Implementation](https://github.com/meta-llama/llama-stack/pull/355) + - [Model context protocol Tool Runtime](https://github.com/meta-llama/llama-stack/pull/665) -2. In addition, if you want to unit test functionality within your provider, feel free to do so. You can find some tests in `tests/` but they aren't well-supported so far. +3. **Register Your Provider** + - Add your provider to the appropriate {repopath}`Registry::llama_stack/providers/registry/` + - Specify any required pip dependencies -3. Test with a client-server Llama Stack setup. (a) Start a Llama Stack server with your own distribution which includes the new provider. (b) Send a client request to the server. See `llama_stack/apis//client.py` for how this is done. These client scripts can serve as lightweight tests. +4. **Integration** + - Update the run.yaml file to include your provider + - To make your provider a default option or create a new distribution, look at the teamplates in {repopath}`llama_stack/templates/` and run {repopath}`llama_stack/scripts/distro_codegen.py` + - Example PRs: + - [Adding Model Context Protocol Tool Runtime](https://github.com/meta-llama/llama-stack/pull/816) -You can find more complex client scripts [llama-stack-apps](https://github.com/meta-llama/llama-stack-apps/tree/main) repo. Note down which scripts works and do not work with your distribution. +## Testing Guidelines -## Submit your PR +### 1. Integration Testing +- Create integration tests that use real provider instances and configurations +- For remote services, test actual API interactions +- Avoid mocking at the provider level +- Reference examples in {repopath}`tests/client-sdk` -After you have fully tested your newly added API provider, submit a PR with the attached test plan. You must have a Test Plan in the summary section of your PR. +### 2. Unit Testing (Optional) +- Add unit tests for provider-specific functionality +- See examples in {repopath}`llama_stack/providers/tests/inference/test_text_inference.py` + +### 3. End-to-End Testing +1. Start a Llama Stack server with your new provider +2. Test using client requests +3. Verify compatibility with existing client scripts in the [llama-stack-apps](https://github.com/meta-llama/llama-stack-apps/tree/main) repository +4. Document which scripts are compatible with your provider + +## Submitting Your PR + +1. Ensure all tests pass +2. Include a comprehensive test plan in your PR summary +3. Document any known limitations or considerations +4. Submit your pull request for review diff --git a/docs/source/distributions/building_distro.md b/docs/source/distributions/building_distro.md index ea0523d05..5556d4aa1 100644 --- a/docs/source/distributions/building_distro.md +++ b/docs/source/distributions/building_distro.md @@ -4,7 +4,7 @@ This guide will walk you through the steps to get started with building a Llama Stack distribution from scratch with your choice of API providers. -## Llama Stack Build +### Llama Stack Build In order to build your own distribution, we recommend you clone the `llama-stack` repository. @@ -13,29 +13,99 @@ In order to build your own distribution, we recommend you clone the `llama-stack git clone git@github.com:meta-llama/llama-stack.git cd llama-stack pip install -e . - -llama stack build -h ``` +Use the CLI to build your distribution. +The main points to consider are: +1. **Image Type** - Do you want a Conda / venv environment or a Container (eg. Docker) +2. **Template** - Do you want to use a template to build your distribution? or start from scratch ? +3. **Config** - Do you want to use a pre-existing config file to build your distribution? -We will start build our distribution (in the form of a Conda environment, or Docker image). In this step, we will specify: -- `name`: the name for our distribution (e.g. `my-stack`) -- `image_type`: our build image type (`conda | docker`) -- `distribution_spec`: our distribution specs for specifying API providers - - `description`: a short description of the configurations for the distribution - - `providers`: specifies the underlying implementation for serving each API endpoint - - `image_type`: `conda` | `docker` to specify whether to build the distribution in the form of Docker image or Conda environment. +``` +llama stack build -h + +usage: llama stack build [-h] [--config CONFIG] [--template TEMPLATE] [--list-templates | --no-list-templates] [--image-type {conda,container,venv}] [--image-name IMAGE_NAME] + +Build a Llama stack container + +options: + -h, --help show this help message and exit + --config CONFIG Path to a config file to use for the build. You can find example configs in llama_stack/distribution/**/build.yaml. + If this argument is not provided, you will be prompted to enter information interactively + --template TEMPLATE Name of the example template config to use for build. You may use `llama stack build --list-templates` to check out the available templates + --list-templates, --no-list-templates + Show the available templates for building a Llama Stack distribution (default: False) + --image-type {conda,container,venv} + Image Type to use for the build. This can be either conda or container or venv. If not specified, will use the image type from the template config. + --image-name IMAGE_NAME + [for image-type=conda] Name of the conda environment to use for the build. If + not specified, currently active Conda environment will be used. If no Conda + environment is active, you must specify a name. +``` After this step is complete, a file named `-build.yaml` and template file `-run.yaml` will be generated and saved at the output file path specified at the end of the command. ::::{tab-set} +:::{tab-item} Building from a template +To build from alternative API providers, we provide distribution templates for users to get started building a distribution backed by different providers. + +The following command will allow you to see the available templates and their corresponding providers. +``` +llama stack build --list-templates +``` + +``` +------------------------------+-----------------------------------------------------------------------------+ +| Template Name | Description | ++------------------------------+-----------------------------------------------------------------------------+ +| hf-serverless | Use (an external) Hugging Face Inference Endpoint for running LLM inference | ++------------------------------+-----------------------------------------------------------------------------+ +| together | Use Together.AI for running LLM inference | ++------------------------------+-----------------------------------------------------------------------------+ +| vllm-gpu | Use a built-in vLLM engine for running LLM inference | ++------------------------------+-----------------------------------------------------------------------------+ +| experimental-post-training | Experimental template for post training | ++------------------------------+-----------------------------------------------------------------------------+ +| remote-vllm | Use (an external) vLLM server for running LLM inference | ++------------------------------+-----------------------------------------------------------------------------+ +| fireworks | Use Fireworks.AI for running LLM inference | ++------------------------------+-----------------------------------------------------------------------------+ +| tgi | Use (an external) TGI server for running LLM inference | ++------------------------------+-----------------------------------------------------------------------------+ +| bedrock | Use AWS Bedrock for running LLM inference and safety | ++------------------------------+-----------------------------------------------------------------------------+ +| meta-reference-gpu | Use Meta Reference for running LLM inference | ++------------------------------+-----------------------------------------------------------------------------+ +| nvidia | Use NVIDIA NIM for running LLM inference | ++------------------------------+-----------------------------------------------------------------------------+ +| meta-reference-quantized-gpu | Use Meta Reference with fp8, int4 quantization for running LLM inference | ++------------------------------+-----------------------------------------------------------------------------+ +| cerebras | Use Cerebras for running LLM inference | ++------------------------------+-----------------------------------------------------------------------------+ +| ollama | Use (an external) Ollama server for running LLM inference | ++------------------------------+-----------------------------------------------------------------------------+ +| hf-endpoint | Use (an external) Hugging Face Inference Endpoint for running LLM inference | ++------------------------------+-----------------------------------------------------------------------------+ +``` + +You may then pick a template to build your distribution with providers fitted to your liking. + +For example, to build a distribution with TGI as the inference provider, you can run: +``` +$ llama stack build --template tgi +... +You can now edit ~/.llama/distributions/llamastack-tgi/tgi-run.yaml and run `llama stack run ~/.llama/distributions/llamastack-tgi/tgi-run.yaml` +``` +::: :::{tab-item} Building from Scratch -- For a new user, we could start off with running `llama stack build` which will allow you to a interactively enter wizard where you will be prompted to enter build configurations. +If the provided templates do not fit your use case, you could start off with running `llama stack build` which will allow you to a interactively enter wizard where you will be prompted to enter build configurations. + +It would be best to start with a template and understand the structure of the config file and the various concepts ( APIS, providers, resources, etc.) before starting from scratch. ``` llama stack build > Enter a name for your Llama Stack (e.g. my-local-stack): my-stack -> Enter the image type you want your Llama Stack to be built as (docker or conda): conda +> Enter the image type you want your Llama Stack to be built as (container or conda): conda Llama Stack is composed of several APIs working together. Let's select the provider types (implementations) you want to use for these APIs. @@ -57,292 +127,6 @@ You can now edit ~/.llama/distributions/llamastack-my-local-stack/my-local-stack ``` ::: -:::{tab-item} Building from a template -- To build from alternative API providers, we provide distribution templates for users to get started building a distribution backed by different providers. - -The following command will allow you to see the available templates and their corresponding providers. -``` -llama stack build --list-templates -``` - -``` -+------------------------------+----------------------------------------+-----------------------------------------------------------------------------+ -| Template Name | Providers | Description | -+------------------------------+----------------------------------------+-----------------------------------------------------------------------------+ -| tgi | { | Use (an external) TGI server for running LLM inference | -| | "inference": [ | | -| | "remote::tgi" | | -| | ], | | -| | "memory": [ | | -| | "inline::faiss", | | -| | "remote::chromadb", | | -| | "remote::pgvector" | | -| | ], | | -| | "safety": [ | | -| | "inline::llama-guard" | | -| | ], | | -| | "agents": [ | | -| | "inline::meta-reference" | | -| | ], | | -| | "telemetry": [ | | -| | "inline::meta-reference" | | -| | ] | | -| | } | | -+------------------------------+----------------------------------------+-----------------------------------------------------------------------------+ -| remote-vllm | { | Use (an external) vLLM server for running LLM inference | -| | "inference": [ | | -| | "remote::vllm" | | -| | ], | | -| | "memory": [ | | -| | "inline::faiss", | | -| | "remote::chromadb", | | -| | "remote::pgvector" | | -| | ], | | -| | "safety": [ | | -| | "inline::llama-guard" | | -| | ], | | -| | "agents": [ | | -| | "inline::meta-reference" | | -| | ], | | -| | "telemetry": [ | | -| | "inline::meta-reference" | | -| | ] | | -| | } | | -+------------------------------+----------------------------------------+-----------------------------------------------------------------------------+ -| vllm-gpu | { | Use a built-in vLLM engine for running LLM inference | -| | "inference": [ | | -| | "inline::vllm" | | -| | ], | | -| | "memory": [ | | -| | "inline::faiss", | | -| | "remote::chromadb", | | -| | "remote::pgvector" | | -| | ], | | -| | "safety": [ | | -| | "inline::llama-guard" | | -| | ], | | -| | "agents": [ | | -| | "inline::meta-reference" | | -| | ], | | -| | "telemetry": [ | | -| | "inline::meta-reference" | | -| | ] | | -| | } | | -+------------------------------+----------------------------------------+-----------------------------------------------------------------------------+ -| meta-reference-quantized-gpu | { | Use Meta Reference with fp8, int4 quantization for running LLM inference | -| | "inference": [ | | -| | "inline::meta-reference-quantized" | | -| | ], | | -| | "memory": [ | | -| | "inline::faiss", | | -| | "remote::chromadb", | | -| | "remote::pgvector" | | -| | ], | | -| | "safety": [ | | -| | "inline::llama-guard" | | -| | ], | | -| | "agents": [ | | -| | "inline::meta-reference" | | -| | ], | | -| | "telemetry": [ | | -| | "inline::meta-reference" | | -| | ] | | -| | } | | -+------------------------------+----------------------------------------+-----------------------------------------------------------------------------+ -| meta-reference-gpu | { | Use Meta Reference for running LLM inference | -| | "inference": [ | | -| | "inline::meta-reference" | | -| | ], | | -| | "memory": [ | | -| | "inline::faiss", | | -| | "remote::chromadb", | | -| | "remote::pgvector" | | -| | ], | | -| | "safety": [ | | -| | "inline::llama-guard" | | -| | ], | | -| | "agents": [ | | -| | "inline::meta-reference" | | -| | ], | | -| | "telemetry": [ | | -| | "inline::meta-reference" | | -| | ] | | -| | } | | -+------------------------------+----------------------------------------+-----------------------------------------------------------------------------+ -| hf-serverless | { | Use (an external) Hugging Face Inference Endpoint for running LLM inference | -| | "inference": [ | | -| | "remote::hf::serverless" | | -| | ], | | -| | "memory": [ | | -| | "inline::faiss", | | -| | "remote::chromadb", | | -| | "remote::pgvector" | | -| | ], | | -| | "safety": [ | | -| | "inline::llama-guard" | | -| | ], | | -| | "agents": [ | | -| | "inline::meta-reference" | | -| | ], | | -| | "telemetry": [ | | -| | "inline::meta-reference" | | -| | ] | | -| | } | | -+------------------------------+----------------------------------------+-----------------------------------------------------------------------------+ -| together | { | Use Together.AI for running LLM inference | -| | "inference": [ | | -| | "remote::together" | | -| | ], | | -| | "memory": [ | | -| | "inline::faiss", | | -| | "remote::chromadb", | | -| | "remote::pgvector" | | -| | ], | | -| | "safety": [ | | -| | "inline::llama-guard" | | -| | ], | | -| | "agents": [ | | -| | "inline::meta-reference" | | -| | ], | | -| | "telemetry": [ | | -| | "inline::meta-reference" | | -| | ] | | -| | } | | -+------------------------------+----------------------------------------+-----------------------------------------------------------------------------+ -| ollama | { | Use (an external) Ollama server for running LLM inference | -| | "inference": [ | | -| | "remote::ollama" | | -| | ], | | -| | "memory": [ | | -| | "inline::faiss", | | -| | "remote::chromadb", | | -| | "remote::pgvector" | | -| | ], | | -| | "safety": [ | | -| | "inline::llama-guard" | | -| | ], | | -| | "agents": [ | | -| | "inline::meta-reference" | | -| | ], | | -| | "telemetry": [ | | -| | "inline::meta-reference" | | -| | ] | | -| | } | | -+------------------------------+----------------------------------------+-----------------------------------------------------------------------------+ -| bedrock | { | Use AWS Bedrock for running LLM inference and safety | -| | "inference": [ | | -| | "remote::bedrock" | | -| | ], | | -| | "memory": [ | | -| | "inline::faiss", | | -| | "remote::chromadb", | | -| | "remote::pgvector" | | -| | ], | | -| | "safety": [ | | -| | "remote::bedrock" | | -| | ], | | -| | "agents": [ | | -| | "inline::meta-reference" | | -| | ], | | -| | "telemetry": [ | | -| | "inline::meta-reference" | | -| | ] | | -| | } | | -+------------------------------+----------------------------------------+-----------------------------------------------------------------------------+ -| hf-endpoint | { | Use (an external) Hugging Face Inference Endpoint for running LLM inference | -| | "inference": [ | | -| | "remote::hf::endpoint" | | -| | ], | | -| | "memory": [ | | -| | "inline::faiss", | | -| | "remote::chromadb", | | -| | "remote::pgvector" | | -| | ], | | -| | "safety": [ | | -| | "inline::llama-guard" | | -| | ], | | -| | "agents": [ | | -| | "inline::meta-reference" | | -| | ], | | -| | "telemetry": [ | | -| | "inline::meta-reference" | | -| | ] | | -| | } | | -+------------------------------+----------------------------------------+-----------------------------------------------------------------------------+ -| fireworks | { | Use Fireworks.AI for running LLM inference | -| | "inference": [ | | -| | "remote::fireworks" | | -| | ], | | -| | "memory": [ | | -| | "inline::faiss", | | -| | "remote::chromadb", | | -| | "remote::pgvector" | | -| | ], | | -| | "safety": [ | | -| | "inline::llama-guard" | | -| | ], | | -| | "agents": [ | | -| | "inline::meta-reference" | | -| | ], | | -| | "telemetry": [ | | -| | "inline::meta-reference" | | -| | ] | | -| | } | | -+------------------------------+----------------------------------------+-----------------------------------------------------------------------------+ -| cerebras | { | Use Cerebras for running LLM inference | -| | "inference": [ | | -| | "remote::cerebras" | | -| | ], | | -| | "memory": [ | | -| | "inline::faiss", | | -| | "remote::chromadb", | | -| | "remote::pgvector" | | -| | ], | | -| | "safety": [ | | -| | "inline::llama-guard" | | -| | ], | | -| | "agents": [ | | -| | "inline::meta-reference" | | -| | ], | | -| | "telemetry": [ | | -| | "inline::meta-reference" | | -| | ] | | -| | } | | -+------------------------------+----------------------------------------+-----------------------------------------------------------------------------+ -| cerebras | { | Use SambaNova.ai for running LLM inference | -| | "inference": [ | | -| | "remote::sambanova" | | -| | ], | | -| | "safety": [ | | -| | "inline::llama-guard" | | -| | ], | | -| | "memory": [ | | -| | "inline::meta-reference" | | -| | ], | | -| | "agents": [ | | -| | "inline::meta-reference" | | -| | ], | | -| | "telemetry": [ | | -| | "inline::meta-reference" | | -| | ] | | -| | } | | -+------------------------------+----------------------------------------+-----------------------------------------------------------------------------+ -``` - -You may then pick a template to build your distribution with providers fitted to your liking. - -For example, to build a distribution with TGI as the inference provider, you can run: -``` -llama stack build --template tgi -``` - -``` -$ llama stack build --template tgi -... -You can now edit ~/.llama/distributions/llamastack-tgi/tgi-run.yaml and run `llama stack run ~/.llama/distributions/llamastack-tgi/tgi-run.yaml` -``` -::: - :::{tab-item} Building from a pre-existing build config file - In addition to templates, you may customize the build to your liking through editing config files and build from config files with the following command. @@ -368,35 +152,39 @@ llama stack build --config llama_stack/templates/ollama/build.yaml ``` ::: -:::{tab-item} Building Docker +:::{tab-item} Building Container > [!TIP] -> Podman is supported as an alternative to Docker. Set `DOCKER_BINARY` to `podman` in your environment to use Podman. +> Podman is supported as an alternative to Docker. Set `CONTAINER_BINARY` to `podman` in your environment to use Podman. -To build a docker image, you may start off from a template and use the `--image-type docker` flag to specify `docker` as the build image type. +To build a container image, you may start off from a template and use the `--image-type container` flag to specify `container` as the build image type. ``` -llama stack build --template ollama --image-type docker +llama stack build --template ollama --image-type container ``` ``` -$ llama stack build --template ollama --image-type docker +$ llama stack build --template ollama --image-type container ... -Dockerfile created successfully in /tmp/tmp.viA3a3Rdsg/DockerfileFROM python:3.10-slim +Containerfile created successfully in /tmp/tmp.viA3a3Rdsg/ContainerfileFROM python:3.10-slim ... You can now edit ~/meta-llama/llama-stack/tmp/configs/ollama-run.yaml and run `llama stack run ~/meta-llama/llama-stack/tmp/configs/ollama-run.yaml` ``` -After this step is successful, you should be able to find the built docker image and test it with `llama stack run `. +After this step is successful, you should be able to find the built container image and test it with `llama stack run `. ::: :::: -## Running your Stack server +### Running your Stack server Now, let's start the Llama Stack Distribution Server. You will need the YAML configuration file which was written out at the end by the `llama stack build` step. ``` +# Start using template name +llama stack run tgi + +# Start using config file llama stack run ~/.llama/distributions/llamastack-my-local-stack/my-local-stack-run.yaml ``` @@ -422,14 +210,14 @@ Serving API agents POST /agents/step/get POST /agents/turn/get -Listening on ['::', '0.0.0.0']:5000 +Listening on ['::', '0.0.0.0']:8321 INFO: Started server process [2935911] INFO: Waiting for application startup. INFO: Application startup complete. -INFO: Uvicorn running on http://['::', '0.0.0.0']:5000 (Press CTRL+C to quit) +INFO: Uvicorn running on http://['::', '0.0.0.0']:8321 (Press CTRL+C to quit) INFO: 2401:db00:35c:2d2b:face:0:c9:0:54678 - "GET /models/list HTTP/1.1" 200 OK ``` ### Troubleshooting -If you encounter any issues, search through our [GitHub Issues](https://github.com/meta-llama/llama-stack/issues), or file an new issue. +If you encounter any issues, ask questions in our discord or search through our [GitHub Issues](https://github.com/meta-llama/llama-stack/issues), or file an new issue. diff --git a/docs/source/distributions/configuration.md b/docs/source/distributions/configuration.md index 41df26618..d12f584f7 100644 --- a/docs/source/distributions/configuration.md +++ b/docs/source/distributions/configuration.md @@ -70,20 +70,27 @@ Next up is the most critical part: the set of providers that the stack will use ```yaml providers: inference: + # provider_id is a string you can choose freely - provider_id: ollama + # provider_type is a string that specifies the type of provider. + # in this case, the provider for inference is ollama and it is run remotely (outside of the distribution) provider_type: remote::ollama + # config is a dictionary that contains the configuration for the provider. + # in this case, the configuration is the url of the ollama server config: url: ${env.OLLAMA_URL:http://localhost:11434} ``` A few things to note: -- A _provider instance_ is identified with an (identifier, type, configuration) tuple. The identifier is a string you can choose freely. +- A _provider instance_ is identified with an (id, type, configuration) triplet. +- The id is a string you can choose freely. - You can instantiate any number of provider instances of the same type. -- The configuration dictionary is provider-specific. Notice that configuration can reference environment variables (with default values), which are expanded at runtime. When you run a stack server (via docker or via `llama stack run`), you can specify `--env OLLAMA_URL=http://my-server:11434` to override the default value. +- The configuration dictionary is provider-specific. +- Notice that configuration can reference environment variables (with default values), which are expanded at runtime. When you run a stack server (via docker or via `llama stack run`), you can specify `--env OLLAMA_URL=http://my-server:11434` to override the default value. ## Resources -``` Finally, let's look at the `models` section: + ```yaml models: - metadata: {} diff --git a/docs/source/distributions/importing_as_library.md b/docs/source/distributions/importing_as_library.md index 7e15062df..cc7ed1beb 100644 --- a/docs/source/distributions/importing_as_library.md +++ b/docs/source/distributions/importing_as_library.md @@ -1,11 +1,20 @@ # Using Llama Stack as a Library -If you are planning to use an external service for Inference (even Ollama or TGI counts as external), it is often easier to use Llama Stack as a library. This avoids the overhead of setting up a server. For [example](https://github.com/meta-llama/llama-stack-client-python/blob/main/src/llama_stack_client/lib/direct/test.py): +If you are planning to use an external service for Inference (even Ollama or TGI counts as external), it is often easier to use Llama Stack as a library. This avoids the overhead of setting up a server. +```python +# setup +pip install llama-stack +llama stack build --template together --image-type venv +``` ```python -from llama_stack_client.lib.direct.direct import LlamaStackDirectClient +from llama_stack.distribution.library_client import LlamaStackAsLibraryClient -client = await LlamaStackDirectClient.from_template('ollama') +client = LlamaStackAsLibraryClient( + "ollama", + # provider_data is optional, but if you need to pass in any provider specific data, you can do so here. + provider_data = {"tavily_search_api_key": os.environ['TAVILY_SEARCH_API_KEY']} +) await client.initialize() ``` @@ -14,23 +23,12 @@ This will parse your config and set up any inline implementations and remote cli Then, you can access the APIs like `models` and `inference` on the client and call their methods directly: ```python -response = await client.models.list() -print(response) -``` - -```python -response = await client.inference.chat_completion( - messages=[UserMessage(content="What is the capital of France?", role="user")], - model_id="Llama3.1-8B-Instruct", - stream=False, -) -print("\nChat completion response:") -print(response) +response = client.models.list() ``` If you've created a [custom distribution](https://llama-stack.readthedocs.io/en/latest/distributions/building_distro.html), you can also use the run.yaml configuration file directly: ```python -client = await LlamaStackDirectClient.from_config(config_path) -await client.initialize() +client = LlamaStackAsLibraryClient(config_path) +client.initialize() ``` diff --git a/docs/source/distributions/index.md b/docs/source/distributions/index.md index 9b2f46869..64fec543f 100644 --- a/docs/source/distributions/index.md +++ b/docs/source/distributions/index.md @@ -1,40 +1,27 @@ -# Starting a Llama Stack +# Starting a Llama Stack Server + +You can run a Llama Stack server in one of the following ways: + +**As a Library**: + +This is the simplest way to get started. Using Llama Stack as a library means you do not need to start a server. This is especially useful when you are not running inference locally and relying on an external inference service (eg. fireworks, together, groq, etc.) See [Using Llama Stack as a Library](importing_as_library) + + +**Docker**: + +Another simple way to start interacting with Llama Stack is to just spin up docker which is pre-built with all the providers you need. We provide a number of pre-built Docker containers so you can start a Llama Stack server instantly. You can also build your own custom Docker container. Which distribution to choose depends on the hardware you have. See [Selection of a Distribution](distributions/selection) for more details. + + +**Conda**: + +Lastly, if you have a custom or an advanced setup or you are developing on Llama Stackyou can also build a custom Llama Stack server. Using `llama stack build` and `llama stack run` you can build/run a custom Llama Stack server containing the exact combination of providers you wish. We have also provided various templates to make getting started easier. See [Building a Custom Distribution](building_distro) for more details. + + ```{toctree} -:maxdepth: 3 +:maxdepth: 1 :hidden: importing_as_library building_distro configuration ``` - -You can instantiate a Llama Stack in one of the following ways: -- **As a Library**: this is the simplest, especially if you are using an external inference service. See [Using Llama Stack as a Library](importing_as_library) -- **Docker**: we provide a number of pre-built Docker containers so you can start a Llama Stack server instantly. You can also build your own custom Docker container. -- **Conda**: finally, you can build a custom Llama Stack server using `llama stack build` containing the exact combination of providers you wish. We have provided various templates to make getting started easier. - -Which templates / distributions to choose depends on the hardware you have for running LLM inference. - -- **Do you have access to a machine with powerful GPUs?** -If so, we suggest: - - {dockerhub}`distribution-remote-vllm` ([Guide](self_hosted_distro/remote-vllm)) - - {dockerhub}`distribution-meta-reference-gpu` ([Guide](self_hosted_distro/meta-reference-gpu)) - - {dockerhub}`distribution-tgi` ([Guide](self_hosted_distro/tgi)) - -- **Are you running on a "regular" desktop machine?** -If so, we suggest: - - {dockerhub}`distribution-ollama` ([Guide](self_hosted_distro/ollama)) - -- **Do you have an API key for a remote inference provider like Fireworks, Together, etc.?** If so, we suggest: - - {dockerhub}`distribution-together` ([Guide](self_hosted_distro/together)) - - {dockerhub}`distribution-fireworks` ([Guide](self_hosted_distro/fireworks)) - -- **Do you want to run Llama Stack inference on your iOS / Android device** If so, we suggest: - - [iOS SDK](ondevice_distro/ios_sdk) - - [Android](ondevice_distro/android_sdk) - -- **Do you want a hosted Llama Stack endpoint?** If so, we suggest: - - [Remote-Hosted Llama Stack Endpoints](remote_hosted_distro/index) - - -You can also build your own [custom distribution](building_distro). diff --git a/docs/source/distributions/ondevice_distro/ios_sdk.md b/docs/source/distributions/ondevice_distro/ios_sdk.md index 0c3cf09af..ffaf74533 100644 --- a/docs/source/distributions/ondevice_distro/ios_sdk.md +++ b/docs/source/distributions/ondevice_distro/ios_sdk.md @@ -1,6 +1,3 @@ ---- -orphan: true ---- # iOS SDK We offer both remote and on-device use of Llama Stack in Swift via two components: @@ -27,7 +24,7 @@ If you don't want to run inference on-device, then you can connect to any hosted ```swift import LlamaStackClient -let agents = RemoteAgents(url: URL(string: "http://localhost:5000")!) +let agents = RemoteAgents(url: URL(string: "http://localhost:8321")!) let request = Components.Schemas.CreateAgentTurnRequest( agent_id: agentId, messages: [ diff --git a/docs/source/distributions/remote_hosted_distro/index.md b/docs/source/distributions/remote_hosted_distro/index.md index 0f86bf73f..2fbe381af 100644 --- a/docs/source/distributions/remote_hosted_distro/index.md +++ b/docs/source/distributions/remote_hosted_distro/index.md @@ -1,6 +1,3 @@ ---- -orphan: true ---- # Remote-Hosted Distributions Remote-Hosted distributions are available endpoints serving Llama Stack API that you can directly connect to. diff --git a/docs/source/distributions/remote_hosted_distro/nvidia.md b/docs/source/distributions/remote_hosted_distro/nvidia.md new file mode 100644 index 000000000..61b41b1d9 --- /dev/null +++ b/docs/source/distributions/remote_hosted_distro/nvidia.md @@ -0,0 +1,73 @@ +# NVIDIA Distribution + +The `llamastack/distribution-nvidia` distribution consists of the following provider configurations. + +| API | Provider(s) | +|-----|-------------| +| agents | `inline::meta-reference` | +| datasetio | `remote::huggingface`, `inline::localfs` | +| eval | `inline::meta-reference` | +| inference | `remote::nvidia` | +| safety | `inline::llama-guard` | +| scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` | +| telemetry | `inline::meta-reference` | +| tool_runtime | `remote::brave-search`, `remote::tavily-search`, `inline::code-interpreter`, `inline::rag-runtime`, `remote::model-context-protocol` | +| vector_io | `inline::faiss` | + + +### Environment Variables + +The following environment variables can be configured: + +- `LLAMASTACK_PORT`: Port for the Llama Stack distribution server (default: `5001`) +- `NVIDIA_API_KEY`: NVIDIA API Key (default: ``) + +### Models + +The following models are available by default: + +- `meta-llama/Llama-3-8B-Instruct (meta/llama3-8b-instruct)` +- `meta-llama/Llama-3-70B-Instruct (meta/llama3-70b-instruct)` +- `meta-llama/Llama-3.1-8B-Instruct (meta/llama-3.1-8b-instruct)` +- `meta-llama/Llama-3.1-70B-Instruct (meta/llama-3.1-70b-instruct)` +- `meta-llama/Llama-3.1-405B-Instruct-FP8 (meta/llama-3.1-405b-instruct)` +- `meta-llama/Llama-3.2-1B-Instruct (meta/llama-3.2-1b-instruct)` +- `meta-llama/Llama-3.2-3B-Instruct (meta/llama-3.2-3b-instruct)` +- `meta-llama/Llama-3.2-11B-Vision-Instruct (meta/llama-3.2-11b-vision-instruct)` +- `meta-llama/Llama-3.2-90B-Vision-Instruct (meta/llama-3.2-90b-vision-instruct)` + + +### Prerequisite: API Keys + +Make sure you have access to a NVIDIA API Key. You can get one by visiting [https://build.nvidia.com/](https://build.nvidia.com/). + + +## Running Llama Stack with NVIDIA + +You can do this via Conda (build code) or Docker which has a pre-built image. + +### Via Docker + +This method allows you to get started quickly without having to build the distribution code. + +```bash +LLAMA_STACK_PORT=5001 +docker run \ + -it \ + -p $LLAMA_STACK_PORT:$LLAMA_STACK_PORT \ + -v ./run.yaml:/root/my-run.yaml \ + llamastack/distribution-nvidia \ + --yaml-config /root/my-run.yaml \ + --port $LLAMA_STACK_PORT \ + --env NVIDIA_API_KEY=$NVIDIA_API_KEY +``` + +### Via Conda + +```bash +llama stack build --template nvidia --image-type conda +llama stack run ./run.yaml \ + --port 5001 \ + --env NVIDIA_API_KEY=$NVIDIA_API_KEY + --env INFERENCE_MODEL=$INFERENCE_MODEL +``` diff --git a/docs/source/distributions/selection.md b/docs/source/distributions/selection.md new file mode 100644 index 000000000..08c3e985a --- /dev/null +++ b/docs/source/distributions/selection.md @@ -0,0 +1,56 @@ +# List of Distributions + +Here are a list of distributions you can use to start a Llama Stack server that are provided out of the box. + +## Selection of a Distribution / Template + +Which templates / distributions to choose depends on the hardware you have for running LLM inference. + +- **Do you want a hosted Llama Stack endpoint?** If so, we suggest leveraging our partners who host Llama Stack endpoints. Namely, _fireworks.ai_ and _together.xyz_. + - Read more about it here - [Remote-Hosted Endpoints](remote_hosted_distro/index). + + +- **Do you have access to machines with GPUs?** If you wish to run Llama Stack locally or on a cloud instance and host your own Llama Stack endpoint, we suggest: + - {dockerhub}`distribution-remote-vllm` ([Guide](self_hosted_distro/remote-vllm)) + - {dockerhub}`distribution-meta-reference-gpu` ([Guide](self_hosted_distro/meta-reference-gpu)) + - {dockerhub}`distribution-tgi` ([Guide](self_hosted_distro/tgi)) + - {dockerhub}`distribution-nvidia` ([Guide](self_hosted_distro/nvidia)) + +- **Are you running on a "regular" desktop or laptop ?** We suggest using the ollama templte for quick prototyping and get started without having to worry about needing GPUs. + - {dockerhub}`distribution-ollama` ([link](self_hosted_distro/ollama)) + +- **Do you have an API key for a remote inference provider like Fireworks, Together, etc.?** If so, we suggest: + - {dockerhub}`distribution-together` ([Guide](self_hosted_distro/together)) + - {dockerhub}`distribution-fireworks` ([Guide](self_hosted_distro/fireworks)) + +- **Do you want to run Llama Stack inference on your iOS / Android device** Lastly, we also provide templates for running Llama Stack inference on your iOS / Android device: + - [iOS SDK](ondevice_distro/ios_sdk) + - [Android](ondevice_distro/android_sdk) + + +- **If none of the above fit your needs, you can also build your own [custom distribution](building_distro).** + +### Distribution Details + +```{toctree} +:maxdepth: 1 + +remote_hosted_distro/index +self_hosted_distro/remote-vllm +self_hosted_distro/meta-reference-gpu +self_hosted_distro/tgi +self_hosted_distro/nvidia +self_hosted_distro/ollama +self_hosted_distro/together +self_hosted_distro/fireworks +ondevice_distro/index +``` + +### On-Device Distributions + +```{toctree} +:maxdepth: 1 + +ondevice_distro/ios_sdk +ondevice_distro/android_sdk +``` diff --git a/docs/source/distributions/self_hosted_distro/bedrock.md b/docs/source/distributions/self_hosted_distro/bedrock.md index 71adfad09..f9a9f29cd 100644 --- a/docs/source/distributions/self_hosted_distro/bedrock.md +++ b/docs/source/distributions/self_hosted_distro/bedrock.md @@ -15,11 +15,11 @@ The `llamastack/distribution-bedrock` distribution consists of the following pro | datasetio | `remote::huggingface`, `inline::localfs` | | eval | `inline::meta-reference` | | inference | `remote::bedrock` | -| memory | `inline::faiss`, `remote::chromadb`, `remote::pgvector` | | safety | `remote::bedrock` | | scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` | | telemetry | `inline::meta-reference` | -| tool_runtime | `remote::brave-search`, `remote::tavily-search`, `inline::code-interpreter`, `inline::memory-runtime` | +| tool_runtime | `remote::brave-search`, `remote::tavily-search`, `inline::code-interpreter`, `inline::rag-runtime`, `remote::model-context-protocol` | +| vector_io | `inline::faiss`, `remote::chromadb`, `remote::pgvector` | diff --git a/docs/source/distributions/self_hosted_distro/cerebras.md b/docs/source/distributions/self_hosted_distro/cerebras.md index 7ebcdfb94..a44e6287a 100644 --- a/docs/source/distributions/self_hosted_distro/cerebras.md +++ b/docs/source/distributions/self_hosted_distro/cerebras.md @@ -1,25 +1,18 @@ ---- -orphan: true ---- # Cerebras Distribution -```{toctree} -:maxdepth: 2 -:hidden: - -self -``` - The `llamastack/distribution-cerebras` distribution consists of the following provider configurations. | API | Provider(s) | |-----|-------------| | agents | `inline::meta-reference` | +| datasetio | `remote::huggingface`, `inline::localfs` | +| eval | `inline::meta-reference` | | inference | `remote::cerebras` | -| memory | `inline::meta-reference` | | safety | `inline::llama-guard` | +| scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` | | telemetry | `inline::meta-reference` | -| tool_runtime | `remote::brave-search`, `remote::tavily-search`, `inline::code-interpreter`, `inline::memory-runtime` | +| tool_runtime | `remote::brave-search`, `remote::tavily-search`, `inline::code-interpreter`, `inline::rag-runtime` | +| vector_io | `inline::faiss`, `remote::chromadb`, `remote::pgvector` | ### Environment Variables diff --git a/docs/source/distributions/self_hosted_distro/dell-tgi.md b/docs/source/distributions/self_hosted_distro/dell-tgi.md index 705bf2fa7..cf0c02983 100644 --- a/docs/source/distributions/self_hosted_distro/dell-tgi.md +++ b/docs/source/distributions/self_hosted_distro/dell-tgi.md @@ -41,7 +41,7 @@ The script will first start up TGI server, then start up Llama Stack distributio INFO: Started server process [1] INFO: Waiting for application startup. INFO: Application startup complete. -INFO: Uvicorn running on http://[::]:5000 (Press CTRL+C to quit) +INFO: Uvicorn running on http://[::]:8321 (Press CTRL+C to quit) ``` To kill the server @@ -65,7 +65,7 @@ registry.dell.huggingface.co/enterprise-dell-inference-meta-llama-meta-llama-3.1 #### Start Llama Stack server pointing to TGI server ``` -docker run --network host -it -p 5000:5000 -v ./run.yaml:/root/my-run.yaml --gpus=all llamastack/distribution-tgi --yaml_config /root/my-run.yaml +docker run --network host -it -p 8321:8321 -v ./run.yaml:/root/my-run.yaml --gpus=all llamastack/distribution-tgi --yaml_config /root/my-run.yaml ``` Make sure in you `run.yaml` file, you inference provider is pointing to the correct TGI server endpoint. E.g. diff --git a/docs/source/distributions/self_hosted_distro/fireworks.md b/docs/source/distributions/self_hosted_distro/fireworks.md index 335309729..453cd746d 100644 --- a/docs/source/distributions/self_hosted_distro/fireworks.md +++ b/docs/source/distributions/self_hosted_distro/fireworks.md @@ -18,11 +18,11 @@ The `llamastack/distribution-fireworks` distribution consists of the following p | datasetio | `remote::huggingface`, `inline::localfs` | | eval | `inline::meta-reference` | | inference | `remote::fireworks` | -| memory | `inline::faiss`, `remote::chromadb`, `remote::pgvector` | | safety | `inline::llama-guard` | | scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` | | telemetry | `inline::meta-reference` | -| tool_runtime | `remote::brave-search`, `remote::tavily-search`, `inline::code-interpreter`, `inline::memory-runtime` | +| tool_runtime | `remote::brave-search`, `remote::tavily-search`, `inline::code-interpreter`, `inline::rag-runtime`, `remote::model-context-protocol` | +| vector_io | `inline::faiss`, `remote::chromadb`, `remote::pgvector` | ### Environment Variables diff --git a/docs/source/distributions/self_hosted_distro/meta-reference-gpu.md b/docs/source/distributions/self_hosted_distro/meta-reference-gpu.md index a89719dea..a371011fe 100644 --- a/docs/source/distributions/self_hosted_distro/meta-reference-gpu.md +++ b/docs/source/distributions/self_hosted_distro/meta-reference-gpu.md @@ -18,11 +18,11 @@ The `llamastack/distribution-meta-reference-gpu` distribution consists of the fo | datasetio | `remote::huggingface`, `inline::localfs` | | eval | `inline::meta-reference` | | inference | `inline::meta-reference` | -| memory | `inline::faiss`, `remote::chromadb`, `remote::pgvector` | | safety | `inline::llama-guard` | | scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` | | telemetry | `inline::meta-reference` | -| tool_runtime | `remote::brave-search`, `remote::tavily-search`, `inline::code-interpreter`, `inline::memory-runtime` | +| tool_runtime | `remote::brave-search`, `remote::tavily-search`, `inline::code-interpreter`, `inline::rag-runtime`, `remote::model-context-protocol` | +| vector_io | `inline::faiss`, `remote::chromadb`, `remote::pgvector` | Note that you need access to nvidia GPUs to run this distribution. This distribution is not compatible with CPU-only machines or machines with AMD GPUs. diff --git a/docs/source/distributions/self_hosted_distro/meta-reference-quantized-gpu.md b/docs/source/distributions/self_hosted_distro/meta-reference-quantized-gpu.md index 26ed5d05b..a32ccb65e 100644 --- a/docs/source/distributions/self_hosted_distro/meta-reference-quantized-gpu.md +++ b/docs/source/distributions/self_hosted_distro/meta-reference-quantized-gpu.md @@ -18,11 +18,11 @@ The `llamastack/distribution-meta-reference-quantized-gpu` distribution consists | datasetio | `remote::huggingface`, `inline::localfs` | | eval | `inline::meta-reference` | | inference | `inline::meta-reference-quantized` | -| memory | `inline::faiss`, `remote::chromadb`, `remote::pgvector` | | safety | `inline::llama-guard` | | scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` | | telemetry | `inline::meta-reference` | -| tool_runtime | `remote::brave-search`, `remote::tavily-search`, `inline::code-interpreter`, `inline::memory-runtime` | +| tool_runtime | `remote::brave-search`, `remote::tavily-search`, `inline::code-interpreter`, `inline::rag-runtime`, `remote::model-context-protocol` | +| vector_io | `inline::faiss`, `remote::chromadb`, `remote::pgvector` | The only difference vs. the `meta-reference-gpu` distribution is that it has support for more efficient inference -- with fp8, int4 quantization, etc. diff --git a/docs/source/distributions/self_hosted_distro/nvidia.md b/docs/source/distributions/self_hosted_distro/nvidia.md new file mode 100644 index 000000000..b86d950dd --- /dev/null +++ b/docs/source/distributions/self_hosted_distro/nvidia.md @@ -0,0 +1,60 @@ +# NVIDIA Distribution + +The `llamastack/distribution-nvidia` distribution consists of the following provider configurations. + +| API | Provider(s) | +|-----|-------------| +| agents | `inline::meta-reference` | +| inference | `remote::nvidia` | +| memory | `inline::faiss`, `remote::chromadb`, `remote::pgvector` | +| safety | `inline::llama-guard` | +| telemetry | `inline::meta-reference` | + + +### Environment Variables + +The following environment variables can be configured: + +- `LLAMASTACK_PORT`: Port for the Llama Stack distribution server (default: `5001`) +- `NVIDIA_API_KEY`: NVIDIA API Key (default: ``) + +### Models + +The following models are available by default: + +- `${env.INFERENCE_MODEL} (None)` + + +### Prerequisite: API Keys + +Make sure you have access to a NVIDIA API Key. You can get one by visiting [https://build.nvidia.com/](https://build.nvidia.com/). + + +## Running Llama Stack with NVIDIA + +You can do this via Conda (build code) or Docker which has a pre-built image. + +### Via Docker + +This method allows you to get started quickly without having to build the distribution code. + +```bash +LLAMA_STACK_PORT=5001 +docker run \ + -it \ + -p $LLAMA_STACK_PORT:$LLAMA_STACK_PORT \ + -v ./run.yaml:/root/my-run.yaml \ + llamastack/distribution-nvidia \ + --yaml-config /root/my-run.yaml \ + --port $LLAMA_STACK_PORT \ + --env NVIDIA_API_KEY=$NVIDIA_API_KEY +``` + +### Via Conda + +```bash +llama stack build --template nvidia --image-type conda +llama stack run ./run.yaml \ + --port 5001 \ + --env NVIDIA_API_KEY=$NVIDIA_API_KEY +``` diff --git a/docs/source/distributions/self_hosted_distro/ollama.md b/docs/source/distributions/self_hosted_distro/ollama.md index e8e5dd397..93f4adfb3 100644 --- a/docs/source/distributions/self_hosted_distro/ollama.md +++ b/docs/source/distributions/self_hosted_distro/ollama.md @@ -1,6 +1,3 @@ ---- -orphan: true ---- # Ollama Distribution ```{toctree} @@ -18,11 +15,11 @@ The `llamastack/distribution-ollama` distribution consists of the following prov | datasetio | `remote::huggingface`, `inline::localfs` | | eval | `inline::meta-reference` | | inference | `remote::ollama` | -| memory | `inline::faiss`, `remote::chromadb`, `remote::pgvector` | | safety | `inline::llama-guard` | | scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` | | telemetry | `inline::meta-reference` | -| tool_runtime | `remote::brave-search`, `remote::tavily-search`, `inline::code-interpreter`, `inline::memory-runtime` | +| tool_runtime | `remote::brave-search`, `remote::tavily-search`, `inline::code-interpreter`, `inline::rag-runtime` | +| vector_io | `inline::faiss`, `remote::chromadb`, `remote::pgvector` | You should use this distribution if you have a regular desktop machine without very powerful GPUs. Of course, if you have powerful GPUs, you can still continue using this distribution since Ollama supports GPU acceleration.### Environment Variables diff --git a/docs/source/distributions/self_hosted_distro/remote-vllm.md b/docs/source/distributions/self_hosted_distro/remote-vllm.md index 98d02725c..1638e9b11 100644 --- a/docs/source/distributions/self_hosted_distro/remote-vllm.md +++ b/docs/source/distributions/self_hosted_distro/remote-vllm.md @@ -1,6 +1,3 @@ ---- -orphan: true ---- # Remote vLLM Distribution ```{toctree} :maxdepth: 2 @@ -14,11 +11,14 @@ The `llamastack/distribution-remote-vllm` distribution consists of the following | API | Provider(s) | |-----|-------------| | agents | `inline::meta-reference` | +| datasetio | `remote::huggingface`, `inline::localfs` | +| eval | `inline::meta-reference` | | inference | `remote::vllm` | -| memory | `inline::faiss`, `remote::chromadb`, `remote::pgvector` | | safety | `inline::llama-guard` | +| scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` | | telemetry | `inline::meta-reference` | -| tool_runtime | `remote::brave-search`, `remote::tavily-search`, `inline::code-interpreter`, `inline::memory-runtime` | +| tool_runtime | `remote::brave-search`, `remote::tavily-search`, `inline::code-interpreter`, `inline::rag-runtime`, `remote::model-context-protocol` | +| vector_io | `inline::faiss`, `remote::chromadb`, `remote::pgvector` | You can use this distribution if you have GPUs and want to run an independent vLLM server container for running inference. diff --git a/docs/source/distributions/self_hosted_distro/tgi.md b/docs/source/distributions/self_hosted_distro/tgi.md index f4f705b12..5a709d0a8 100644 --- a/docs/source/distributions/self_hosted_distro/tgi.md +++ b/docs/source/distributions/self_hosted_distro/tgi.md @@ -1,7 +1,3 @@ ---- -orphan: true ---- - # TGI Distribution ```{toctree} @@ -19,11 +15,11 @@ The `llamastack/distribution-tgi` distribution consists of the following provide | datasetio | `remote::huggingface`, `inline::localfs` | | eval | `inline::meta-reference` | | inference | `remote::tgi` | -| memory | `inline::faiss`, `remote::chromadb`, `remote::pgvector` | | safety | `inline::llama-guard` | | scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` | | telemetry | `inline::meta-reference` | -| tool_runtime | `remote::brave-search`, `remote::tavily-search`, `inline::code-interpreter`, `inline::memory-runtime` | +| tool_runtime | `remote::brave-search`, `remote::tavily-search`, `inline::code-interpreter`, `inline::rag-runtime`, `remote::model-context-protocol` | +| vector_io | `inline::faiss`, `remote::chromadb`, `remote::pgvector` | You can use this distribution if you have GPUs and want to run an independent TGI server container for running inference. diff --git a/docs/source/distributions/self_hosted_distro/together.md b/docs/source/distributions/self_hosted_distro/together.md index 3b476c9bf..707f5be7a 100644 --- a/docs/source/distributions/self_hosted_distro/together.md +++ b/docs/source/distributions/self_hosted_distro/together.md @@ -1,6 +1,3 @@ ---- -orphan: true ---- # Together Distribution ```{toctree} @@ -18,11 +15,11 @@ The `llamastack/distribution-together` distribution consists of the following pr | datasetio | `remote::huggingface`, `inline::localfs` | | eval | `inline::meta-reference` | | inference | `remote::together` | -| memory | `inline::faiss`, `remote::chromadb`, `remote::pgvector` | | safety | `inline::llama-guard` | | scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` | | telemetry | `inline::meta-reference` | -| tool_runtime | `remote::brave-search`, `remote::tavily-search`, `inline::code-interpreter`, `inline::memory-runtime` | +| tool_runtime | `remote::brave-search`, `remote::tavily-search`, `inline::code-interpreter`, `inline::rag-runtime`, `remote::model-context-protocol` | +| vector_io | `inline::faiss`, `remote::chromadb`, `remote::pgvector` | ### Environment Variables diff --git a/docs/source/getting_started/index.md b/docs/source/getting_started/index.md index d7c3fe9e5..aba3de54e 100644 --- a/docs/source/getting_started/index.md +++ b/docs/source/getting_started/index.md @@ -1,26 +1,24 @@ # Quick Start -In this guide, we'll through how you can use the Llama Stack client SDK to build a simple RAG agent. +In this guide, we'll walk through how you can use the Llama Stack (server and client SDK ) to test a simple RAG agent. -The most critical requirement for running the agent is running inference on the underlying Llama model. Depending on what hardware (GPUs) you have available, you have various options. We will use `Ollama` for this purpose as it is the easiest to get started with and yet robust. +A Llama Stack agent is a simple autonomous system that can perform tasks by combining a Llama model for reasoning with tools (e.g., RAG, web search, code execution, etc.) for taking actions. -First, let's set up some environment variables that we will use in the rest of the guide. Note that if you open up a new terminal, you will need to set these again. +At minimum, an agent requires a Llama model for inference and at least one tool that it can use. + +In Llama Stack, we provide a server exposing multiple APIs. These APIs are backed by implementations from different providers. For this guide, we will use [Ollama](https://ollama.com/) as the inference provider. -```bash -export INFERENCE_MODEL="meta-llama/Llama-3.2-3B-Instruct" -# ollama names this model differently, and we must use the ollama name when loading the model -export OLLAMA_INFERENCE_MODEL="llama3.2:3b-instruct-fp16" -export LLAMA_STACK_PORT=5001 -``` ### 1. Start Ollama ```bash -ollama run $OLLAMA_INFERENCE_MODEL --keepalive 60m +ollama run llama3.2:3b-instruct-fp16 --keepalive 60m ``` By default, Ollama keeps the model loaded in memory for 5 minutes which can be too short. We set the `--keepalive` flag to 60 minutes to ensure the model remains loaded for sometime. +NOTE: If you do not have ollama, you can install it from [here](https://ollama.ai/docs/installation). + ### 2. Start the Llama Stack server @@ -28,6 +26,13 @@ Llama Stack is based on a client-server architecture. It consists of a server wh To get started quickly, we provide various Docker images for the server component that work with different inference providers out of the box. For this guide, we will use `llamastack/distribution-ollama` as the Docker image. +Lets setup some environment variables that we will use in the rest of the guide. +```bash +INFERENCE_MODEL="meta-llama/Llama-3.2-3B-Instruct" +LLAMA_STACK_PORT=8321 +``` + +You can start the server using the following command: ```bash docker run -it \ -p $LLAMA_STACK_PORT:$LLAMA_STACK_PORT \ @@ -45,6 +50,9 @@ Configuration for this is available at `distributions/ollama/run.yaml`. You can interact with the Llama Stack server using various client SDKs. We will use the Python SDK which you can install using the following command. Note that you must be using Python 3.10 or newer: ```bash +yes | conda create -n stack-client python=3.10 +conda activate stack-client + pip install llama-stack-client ``` @@ -76,7 +84,10 @@ client = LlamaStackClient(base_url=f"http://localhost:{os.environ['LLAMA_STACK_P # List available models models = client.models.list() -print(models) +print("--- Available models: ---") +for m in models: + print(f"- {m.identifier}") +print() response = client.inference.chat_completion( model_id=os.environ["INFERENCE_MODEL"], @@ -93,59 +104,83 @@ print(response.completion_message.content) Here is an example of a simple RAG agent that uses the Llama Stack client SDK. ```python -import asyncio import os +from termcolor import cprint -from llama_stack_client import LlamaStackClient from llama_stack_client.lib.agents.agent import Agent from llama_stack_client.lib.agents.event_logger import EventLogger -from llama_stack_client.types import Attachment from llama_stack_client.types.agent_create_params import AgentConfig +from llama_stack_client.types.tool_runtime import DocumentParam as Document +from llama_stack_client import LlamaStackClient -async def run_main(): - urls = ["chat.rst", "llama3.rst", "datasets.rst", "lora_finetune.rst"] - attachments = [ - Attachment( - content=f"https://raw.githubusercontent.com/pytorch/torchtune/main/docs/source/tutorials/{url}", - mime_type="text/plain", - ) - for i, url in enumerate(urls) - ] +# Define the client and point it to the server URL +client = LlamaStackClient(base_url=f"http://localhost:{os.environ['LLAMA_STACK_PORT']}") - client = LlamaStackClient(base_url=f"http://localhost:{os.environ['LLAMA_STACK_PORT']}") - - agent_config = AgentConfig( - model=os.environ["INFERENCE_MODEL"], - instructions="You are a helpful assistant", - tools=[{"type": "memory"}], # enable Memory aka RAG - enable_session_persistence=True, +# Define the documents to be used for RAG +urls = ["chat.rst", "llama3.rst", "datasets.rst", "lora_finetune.rst"] +documents = [ + Document( + document_id=f"num-{i}", + content=f"https://raw.githubusercontent.com/pytorch/torchtune/main/docs/source/tutorials/{url}", + mime_type="text/plain", + metadata={}, ) + for i, url in enumerate(urls) +] - agent = Agent(client, agent_config) - session_id = agent.create_session("test-session") - user_prompts = [ - ( - "I am attaching documentation for Torchtune. Help me answer questions I will ask next.", - attachments, - ), - ( - "What are the top 5 topics that were explained? Only list succinct bullet points.", - None, - ), - ] - for prompt, attachments in user_prompts: - response = agent.create_turn( - messages=[{"role": "user", "content": prompt}], - attachments=attachments, - session_id=session_id, - ) - for log in EventLogger().log(response): - log.print() +# Register a vector database +vector_db_id = "test-vector-db" +client.vector_dbs.register( + vector_db_id=vector_db_id, + embedding_model="all-MiniLM-L6-v2", + embedding_dimension=384, +) +# Insert the documents into the vector database +client.tool_runtime.rag_tool.insert( + documents=documents, + vector_db_id=vector_db_id, + chunk_size_in_tokens=512, +) -if __name__ == "__main__": - asyncio.run(run_main()) +# Create an agent +agent_config = AgentConfig( + # Define the inference model to use + model=os.environ["INFERENCE_MODEL"], + # Define instructions for the agent ( aka system prompt) + instructions="You are a helpful assistant", + # Enable session persistence + enable_session_persistence=False, + # Define tools available to the agent + toolgroups = [ + { + "name": "builtin::memory", + "args" : { + "vector_db_ids": [vector_db_id], + } + } + ], +) + +# Create an agent session +rag_agent = Agent(client, agent_config) +session_id = rag_agent.create_session("test-session") + +# Define a user prompts +user_prompts = [ + "What are the top 5 topics that were explained? Only list succinct bullet points.", +] + +# Run the agent loop by calling the `create_turn` method +for prompt in user_prompts: + cprint(f'User> {prompt}', 'green') + response = rag_agent.create_turn( + messages=[{"role": "user", "content": prompt}], + session_id=session_id, + ) + for log in EventLogger().log(response): + log.print() ``` ## Next Steps diff --git a/docs/source/index.md b/docs/source/index.md index aa2845bae..77afd9d22 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -1,17 +1,15 @@ # Llama Stack -Llama Stack defines and standardizes the set of core building blocks needed to bring generative AI applications to market. These building blocks are presented in the form of interoperable APIs with a broad set of Service Providers providing their implementations. +Llama Stack defines and standardizes the core building blocks needed to bring generative AI applications to market. It provides a unified set of APIs with implementations from leading service providers, enabling seamless transitions between development and production environments. + +We focus on making it easy to build production applications with the Llama model family - from the latest Llama 3.3 to specialized models like Llama Guard for safety. ```{image} ../_static/llama-stack.png :alt: Llama Stack :width: 400px ``` -Our goal is to provide pre-packaged implementations which can be operated in a variety of deployment environments: developers start iterating with Desktops or their mobile devices and can seamlessly transition to on-prem or public cloud deployments. At every point in this transition, the same set of APIs and the same developer experience is available. - -```{note} -The Stack APIs are rapidly improving but still a work-in-progress. We invite feedback as well as direct contributions. -``` +Our goal is to provide pre-packaged implementations (aka "distributions") which can be run in a variety of deployment environments. LlamaStack can assist you in your entire app development lifecycle - start iterating on local, mobile or desktop and seamlessly transition to on-prem or public cloud deployments. At every point in this transition, the same set of APIs and the same developer experience is available. ## Quick Links @@ -45,7 +43,7 @@ A number of "adapters" are available for some popular Inference and Memory (Vect | SambaNova | Hosted | | Y | | | | | Ollama | Single Node | | Y | | | | TGI | Hosted and Single Node | | Y | | | -| [NVIDIA NIM](https://build.nvidia.com/nim?filters=nimType%3Anim_type_run_anywhere&q=llama) | Hosted and Single Node | | Y | | | +| NVIDIA NIM | Hosted and Single Node | | Y | | | | Chroma | Single Node | | | Y | | | | Postgres | Single Node | | | Y | | | | PyTorch ExecuTorch | On-device iOS | Y | Y | | | @@ -55,10 +53,12 @@ A number of "adapters" are available for some popular Inference and Memory (Vect :hidden: :maxdepth: 3 +self introduction/index getting_started/index concepts/index distributions/index +distributions/selection building_applications/index benchmark_evaluations/index playground/index diff --git a/docs/source/introduction/index.md b/docs/source/introduction/index.md index 9c2a70341..beae53158 100644 --- a/docs/source/introduction/index.md +++ b/docs/source/introduction/index.md @@ -19,77 +19,41 @@ Building production AI applications today requires solving multiple challenges: - Changing providers requires significant code changes. -### The Vision: A Universal Stack - +### Our Solution: A Universal Stack ```{image} ../../_static/llama-stack.png :alt: Llama Stack :width: 400px ``` -Llama Stack defines and standardizes the core building blocks needed to bring generative AI applications to market. These building blocks are presented as interoperable APIs with a broad set of Service Providers providing their implementations. +Llama Stack addresses these challenges through a service-oriented, API-first approach: -#### Service-oriented Design -Unlike other frameworks, Llama Stack is built with a service-oriented, REST API-first approach. Such a design not only allows for seamless transitions from local to remote deployments but also forces the design to be more declarative. This restriction can result in a much simpler, robust developer experience. The same code works across different environments: +**Develop Anywhere, Deploy Everywhere** +- Start locally with CPU-only setups +- Move to GPU acceleration when needed +- Deploy to cloud or edge without code changes +- Same APIs and developer experience everywhere -- Local development with CPU-only setups -- Self-hosted with GPU acceleration -- Cloud-hosted on providers like AWS, Fireworks, Together -- On-device for iOS and Android - - -#### Composability -The APIs we design are composable. An Agent abstractly depends on { Inference, Memory, Safety } APIs but does not care about the actual implementation details. Safety itself may require model inference and hence can depend on the Inference API. - -#### Turnkey Solutions - -We provide turnkey solutions for popular deployment scenarios. It should be easy to deploy a Llama Stack server on AWS or in a private data center. Either of these should allow a developer to get started with powerful agentic apps, model evaluations, or fine-tuning services in minutes. - -We have built-in support for critical needs: - -- Safety guardrails and content filtering -- Comprehensive evaluation capabilities +**Production-Ready Building Blocks** +- Pre-built safety guardrails and content filtering +- Built-in RAG and agent capabilities +- Comprehensive evaluation toolkit - Full observability and monitoring -- Provider federation and fallback -#### Focus on Llama Models -As a Meta-initiated project, we explicitly focus on Meta's Llama series of models. Supporting the broad set of open models is no easy task and we want to start with models we understand best. - -#### Supporting the Ecosystem -There is a vibrant ecosystem of Providers which provide efficient inference or scalable vector stores or powerful observability solutions. We want to make sure it is easy for developers to pick and choose the best implementations for their use cases. We also want to make sure it is easy for new Providers to onboard and participate in the ecosystem. - -Additionally, we have designed every element of the Stack such that APIs as well as Resources (like Models) can be federated. - -#### Rich Provider Ecosystem - -```{list-table} -:header-rows: 1 - -* - Provider - - Local - - Self-hosted - - Cloud -* - Inference - - Ollama - - vLLM, TGI - - Fireworks, Together, AWS -* - Memory - - FAISS - - Chroma, pgvector - - Weaviate -* - Safety - - Llama Guard - - - - - AWS Bedrock -``` +**True Provider Independence** +- Swap providers without application changes +- Mix and match best-in-class implementations +- Federation and fallback support +- No vendor lock-in -### Unified API Layer +### Our Philosophy -Llama Stack provides a consistent interface for: +- **Service-Oriented**: REST APIs enforce clean interfaces and enable seamless transitions across different environments. +- **Composability**: Every component is independent but works together seamlessly +- **Production Ready**: Built for real-world applications, not just demos +- **Turnkey Solutions**: Easy to deploy built in solutions for popular deployment scenarios +- **Llama First**: Explicit focus on Meta's Llama models and partnering ecosystem -- **Inference**: Run LLM models efficiently -- **Safety**: Apply content filtering and safety policies -- **Memory**: Store and retrieve knowledge for RAG -- **Agents**: Build multi-step workflows -- **Evaluation**: Test and improve application quality + +With Llama Stack, you can focus on building your application while we handle the infrastructure complexity, essential capabilities, and provider integrations. diff --git a/docs/source/references/evals_reference/index.md b/docs/source/references/evals_reference/index.md index f93b56e64..c01fd69d8 100644 --- a/docs/source/references/evals_reference/index.md +++ b/docs/source/references/evals_reference/index.md @@ -92,9 +92,10 @@ response = client.eval.evaluate_rows( "type": "model", "model": "meta-llama/Llama-3.2-90B-Vision-Instruct", "sampling_params": { - "temperature": 0.0, + "strategy": { + "type": "greedy", + }, "max_tokens": 4096, - "top_p": 0.9, "repeat_penalty": 1.0, }, "system_message": system_message @@ -149,9 +150,10 @@ response = client.eval.evaluate_rows( "type": "model", "model": "meta-llama/Llama-3.2-90B-Vision-Instruct", "sampling_params": { - "temperature": 0.0, + "strategy": { + "type": "greedy", + }, "max_tokens": 4096, - "top_p": 0.9, "repeat_penalty": 1.0, }, } @@ -170,9 +172,9 @@ agent_config = { "model": "meta-llama/Llama-3.1-405B-Instruct", "instructions": "You are a helpful assistant", "sampling_params": { - "strategy": "greedy", - "temperature": 0.0, - "top_p": 0.95, + "strategy": { + "type": "greedy", + }, }, "tools": [ { @@ -318,10 +320,9 @@ The `EvalTaskConfig` are user specified config to define: "type": "model", "model": "Llama3.2-3B-Instruct", "sampling_params": { - "strategy": "greedy", - "temperature": 0, - "top_p": 0.95, - "top_k": 0, + "strategy": { + "type": "greedy", + }, "max_tokens": 0, "repetition_penalty": 1.0 } @@ -337,10 +338,9 @@ The `EvalTaskConfig` are user specified config to define: "type": "model", "model": "Llama3.1-405B-Instruct", "sampling_params": { - "strategy": "greedy", - "temperature": 0, - "top_p": 0.95, - "top_k": 0, + "strategy": { + "type": "greedy", + }, "max_tokens": 0, "repetition_penalty": 1.0 } diff --git a/docs/source/references/llama_cli_reference/index.md b/docs/source/references/llama_cli_reference/index.md index a0314644a..f7ac5fe36 100644 --- a/docs/source/references/llama_cli_reference/index.md +++ b/docs/source/references/llama_cli_reference/index.md @@ -214,7 +214,6 @@ llama model describe -m Llama3.2-3B-Instruct | | } | +-----------------------------+----------------------------------+ | Recommended sampling params | { | -| | "strategy": "top_p", | | | "temperature": 1.0, | | | "top_p": 0.9, | | | "top_k": 0 | diff --git a/docs/source/references/llama_stack_client_cli_reference.md b/docs/source/references/llama_stack_client_cli_reference.md index b35aa189d..bc5f3e5e6 100644 --- a/docs/source/references/llama_stack_client_cli_reference.md +++ b/docs/source/references/llama_stack_client_cli_reference.md @@ -23,8 +23,8 @@ subcommands: ```bash $ llama-stack-client configure > Enter the host name of the Llama Stack distribution server: localhost -> Enter the port number of the Llama Stack distribution server: 5000 -Done! You can now use the Llama Stack Client CLI with endpoint http://localhost:5000 +> Enter the port number of the Llama Stack distribution server: 8321 +Done! You can now use the Llama Stack Client CLI with endpoint http://localhost:8321 ``` ### `llama-stack-client providers list` @@ -200,10 +200,9 @@ Example eval_task_config.json: "type": "model", "model": "Llama3.1-405B-Instruct", "sampling_params": { - "strategy": "greedy", - "temperature": 0, - "top_p": 0.95, - "top_k": 0, + "strategy": { + "type": "greedy" + }, "max_tokens": 0, "repetition_penalty": 1.0 } diff --git a/docs/to_situate/developer_cookbook.md b/docs/to_situate/developer_cookbook.md deleted file mode 100644 index 56ebd7a76..000000000 --- a/docs/to_situate/developer_cookbook.md +++ /dev/null @@ -1,41 +0,0 @@ -# Llama Stack Developer Cookbook - -Based on your developer needs, below are references to guides to help you get started. - -### Hosted Llama Stack Endpoint -* Developer Need: I want to connect to a Llama Stack endpoint to build my applications. -* Effort: 1min -* Guide: - - Checkout our [DeepLearning course](https://www.deeplearning.ai/short-courses/introducing-multimodal-llama-3-2) on building with Llama Stack apps on pre-hosted Llama Stack endpoint. - - -### Local meta-reference Llama Stack Server -* Developer Need: I want to start a local Llama Stack server with my GPU using meta-reference implementations. -* Effort: 5min -* Guide: - - Please see our [meta-reference-gpu](https://llama-stack.readthedocs.io/en/latest/distributions/self_hosted_distro/meta-reference-gpu.html) on starting up a meta-reference Llama Stack server. - -### Llama Stack Server with Remote Providers -* Developer need: I want a Llama Stack distribution with a remote provider. -* Effort: 10min -* Guide - - Please see our [Distributions Guide](https://llama-stack.readthedocs.io/en/latest/concepts/index.html#distributions) on starting up distributions with remote providers. - - -### On-Device (iOS) Llama Stack -* Developer Need: I want to use Llama Stack on-Device -* Effort: 1.5hr -* Guide: - - Please see our [iOS Llama Stack SDK](./ios_sdk.md) implementations - -### Assemble your own Llama Stack Distribution -* Developer Need: I want to assemble my own distribution with API providers to my likings -* Effort: 30min -* Guide - - Please see our [Building Distribution](./building_distro.md) guide for assembling your own Llama Stack distribution with your choice of API providers. - -### Adding a New API Provider -* Developer Need: I want to add a new API provider to Llama Stack. -* Effort: 3hr -* Guide - - Please see our [Adding a New API Provider](https://llama-stack.readthedocs.io/en/latest/contributing/new_api_provider.html) guide for adding a new API provider. diff --git a/docs/zero_to_hero_guide/01_Local_Cloud_Inference101.ipynb b/docs/zero_to_hero_guide/01_Local_Cloud_Inference101.ipynb index bdfd3520f..39644ee51 100644 --- a/docs/zero_to_hero_guide/01_Local_Cloud_Inference101.ipynb +++ b/docs/zero_to_hero_guide/01_Local_Cloud_Inference101.ipynb @@ -32,8 +32,8 @@ "outputs": [], "source": [ "HOST = \"localhost\" # Replace with your host\n", - "LOCAL_PORT = 5000 # Replace with your local distro port\n", - "CLOUD_PORT = 5001 # Replace with your cloud distro port" + "LOCAL_PORT = 8321 # Replace with your local distro port\n", + "CLOUD_PORT = 8322 # Replace with your cloud distro port" ] }, { @@ -43,7 +43,7 @@ "source": [ "#### 2. Set Up Local and Cloud Clients\n", "\n", - "Initialize both clients, specifying the `base_url` for each instance. In this case, we have the local distribution running on `http://localhost:5000` and the cloud distribution running on `http://localhost:5001`.\n" + "Initialize both clients, specifying the `base_url` for each instance. In this case, we have the local distribution running on `http://localhost:8321` and the cloud distribution running on `http://localhost:5001`.\n" ] }, { diff --git a/docs/zero_to_hero_guide/04_Tool_Calling101.ipynb b/docs/zero_to_hero_guide/04_Tool_Calling101.ipynb index 4f0d2e887..4c278493b 100644 --- a/docs/zero_to_hero_guide/04_Tool_Calling101.ipynb +++ b/docs/zero_to_hero_guide/04_Tool_Calling101.ipynb @@ -26,27 +26,28 @@ "metadata": {}, "outputs": [], "source": [ - "import os\n", - "import requests\n", - "import json\n", "import asyncio\n", - "import nest_asyncio\n", + "import json\n", + "import os\n", "from typing import Dict, List\n", + "\n", + "import nest_asyncio\n", + "import requests\n", "from dotenv import load_dotenv\n", "from llama_stack_client import LlamaStackClient\n", - "from llama_stack_client.lib.agents.custom_tool import CustomTool\n", - "from llama_stack_client.types.shared.tool_response_message import ToolResponseMessage\n", - "from llama_stack_client.types import CompletionMessage\n", "from llama_stack_client.lib.agents.agent import Agent\n", + "from llama_stack_client.lib.agents.custom_tool import CustomTool\n", "from llama_stack_client.lib.agents.event_logger import EventLogger\n", + "from llama_stack_client.types import CompletionMessage\n", "from llama_stack_client.types.agent_create_params import AgentConfig\n", + "from llama_stack_client.types.shared.tool_response_message import ToolResponseMessage\n", "\n", "# Allow asyncio to run in Jupyter Notebook\n", "nest_asyncio.apply()\n", "\n", - "HOST='localhost'\n", - "PORT=5001\n", - "MODEL_NAME='meta-llama/Llama-3.2-3B-Instruct'" + "HOST = \"localhost\"\n", + "PORT = 5001\n", + "MODEL_NAME = \"meta-llama/Llama-3.2-3B-Instruct\"\n" ] }, { @@ -69,7 +70,7 @@ "outputs": [], "source": [ "load_dotenv()\n", - "BRAVE_SEARCH_API_KEY = os.environ['BRAVE_SEARCH_API_KEY']" + "BRAVE_SEARCH_API_KEY = os.environ[\"BRAVE_SEARCH_API_KEY\"]\n" ] }, { @@ -118,7 +119,7 @@ " cleaned = {k: v for k, v in results[idx].items() if k in selected_keys}\n", " clean_response.append(cleaned)\n", "\n", - " return {\"query\": query, \"top_k\": clean_response}" + " return {\"query\": query, \"top_k\": clean_response}\n" ] }, { @@ -157,25 +158,29 @@ " for message in messages:\n", " if isinstance(message, CompletionMessage) and message.tool_calls:\n", " for tool_call in message.tool_calls:\n", - " if 'query' in tool_call.arguments:\n", - " query = tool_call.arguments['query']\n", + " if \"query\" in tool_call.arguments:\n", + " query = tool_call.arguments[\"query\"]\n", " call_id = tool_call.call_id\n", "\n", " if query:\n", " search_result = await self.run_impl(query)\n", - " return [ToolResponseMessage(\n", - " call_id=call_id,\n", - " role=\"ipython\",\n", - " content=self._format_response_for_agent(search_result),\n", - " tool_name=\"brave_search\"\n", - " )]\n", + " return [\n", + " ToolResponseMessage(\n", + " call_id=call_id,\n", + " role=\"ipython\",\n", + " content=self._format_response_for_agent(search_result),\n", + " tool_name=\"brave_search\",\n", + " )\n", + " ]\n", "\n", - " return [ToolResponseMessage(\n", - " call_id=\"no_call_id\",\n", - " role=\"ipython\",\n", - " content=\"No query provided.\",\n", - " tool_name=\"brave_search\"\n", - " )]\n", + " return [\n", + " ToolResponseMessage(\n", + " call_id=\"no_call_id\",\n", + " role=\"ipython\",\n", + " content=\"No query provided.\",\n", + " tool_name=\"brave_search\",\n", + " )\n", + " ]\n", "\n", " def _format_response_for_agent(self, search_result):\n", " parsed_result = json.loads(search_result)\n", @@ -186,7 +191,7 @@ " f\" URL: {result.get('url', 'No URL')}\\n\"\n", " f\" Description: {result.get('description', 'No Description')}\\n\\n\"\n", " )\n", - " return formatted_result" + " return formatted_result\n" ] }, { @@ -209,7 +214,7 @@ "async def execute_search(query: str):\n", " web_search_tool = WebSearchTool(api_key=BRAVE_SEARCH_API_KEY)\n", " result = await web_search_tool.run_impl(query)\n", - " print(\"Search Results:\", result)" + " print(\"Search Results:\", result)\n" ] }, { @@ -236,7 +241,7 @@ ], "source": [ "query = \"Latest developments in quantum computing\"\n", - "asyncio.run(execute_search(query))" + "asyncio.run(execute_search(query))\n" ] }, { @@ -288,19 +293,17 @@ "\n", " # Initialize custom tool (ensure `WebSearchTool` is defined earlier in the notebook)\n", " webSearchTool = WebSearchTool(api_key=BRAVE_SEARCH_API_KEY)\n", - " \n", + "\n", " # Define the agent configuration, including the model and tool setup\n", " agent_config = AgentConfig(\n", " model=MODEL_NAME,\n", " instructions=\"\"\"You are a helpful assistant that responds to user queries with relevant information and cites sources when available.\"\"\",\n", " sampling_params={\n", - " \"strategy\": \"greedy\",\n", - " \"temperature\": 1.0,\n", - " \"top_p\": 0.9,\n", + " \"strategy\": {\n", + " \"type\": \"greedy\",\n", + " },\n", " },\n", - " tools=[\n", - " webSearchTool.get_tool_definition()\n", - " ],\n", + " tools=[webSearchTool.get_tool_definition()],\n", " tool_choice=\"auto\",\n", " tool_prompt_format=\"python_list\",\n", " input_shields=input_shields,\n", @@ -329,8 +332,9 @@ " async for log in EventLogger().log(response):\n", " log.print()\n", "\n", + "\n", "# Run the function asynchronously in a Jupyter Notebook cell\n", - "await run_main(disable_safety=True)" + "await run_main(disable_safety=True)\n" ] } ], diff --git a/docs/zero_to_hero_guide/07_Agents101.ipynb b/docs/zero_to_hero_guide/07_Agents101.ipynb index 88b73b4cd..04178f3f6 100644 --- a/docs/zero_to_hero_guide/07_Agents101.ipynb +++ b/docs/zero_to_hero_guide/07_Agents101.ipynb @@ -50,8 +50,8 @@ "outputs": [], "source": [ "HOST = \"localhost\" # Replace with your host\n", - "PORT = 5001 # Replace with your port\n", - "MODEL_NAME='meta-llama/Llama-3.2-3B-Instruct'" + "PORT = 5001 # Replace with your port\n", + "MODEL_NAME = \"meta-llama/Llama-3.2-3B-Instruct\"\n" ] }, { @@ -60,10 +60,12 @@ "metadata": {}, "outputs": [], "source": [ - "from dotenv import load_dotenv\n", "import os\n", + "\n", + "from dotenv import load_dotenv\n", + "\n", "load_dotenv()\n", - "BRAVE_SEARCH_API_KEY = os.environ['BRAVE_SEARCH_API_KEY']" + "BRAVE_SEARCH_API_KEY = os.environ[\"BRAVE_SEARCH_API_KEY\"]\n" ] }, { @@ -104,20 +106,22 @@ ], "source": [ "import os\n", + "\n", "from llama_stack_client import LlamaStackClient\n", "from llama_stack_client.lib.agents.agent import Agent\n", "from llama_stack_client.lib.agents.event_logger import EventLogger\n", "from llama_stack_client.types.agent_create_params import AgentConfig\n", "\n", + "\n", "async def agent_example():\n", " client = LlamaStackClient(base_url=f\"http://{HOST}:{PORT}\")\n", " agent_config = AgentConfig(\n", " model=MODEL_NAME,\n", " instructions=\"You are a helpful assistant! If you call builtin tools like brave search, follow the syntax brave_search.call(…)\",\n", " sampling_params={\n", - " \"strategy\": \"greedy\",\n", - " \"temperature\": 1.0,\n", - " \"top_p\": 0.9,\n", + " \"strategy\": {\n", + " \"type\": \"greedy\",\n", + " },\n", " },\n", " tools=[\n", " {\n", @@ -157,7 +161,7 @@ " log.print()\n", "\n", "\n", - "await agent_example()" + "await agent_example()\n" ] }, { diff --git a/docs/zero_to_hero_guide/README.md b/docs/zero_to_hero_guide/README.md index f96ae49ce..c4803a1d6 100644 --- a/docs/zero_to_hero_guide/README.md +++ b/docs/zero_to_hero_guide/README.md @@ -157,7 +157,15 @@ curl http://localhost:$LLAMA_STACK_PORT/alpha/inference/chat-completion {"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Write me a 2-sentence poem about the moon"} ], - "sampling_params": {"temperature": 0.7, "seed": 42, "max_tokens": 512} + "sampling_params": { + "strategy": { + "type": "top_p", + "temperatrue": 0.7, + "top_p": 0.95, + }, + "seed": 42, + "max_tokens": 512 + } } EOF ``` diff --git a/docs/zero_to_hero_guide/Tool_Calling101_Using_Together's_Llama_Stack_Server.ipynb b/docs/zero_to_hero_guide/Tool_Calling101_Using_Together's_Llama_Stack_Server.ipynb index b21f3d64c..68e781018 100644 --- a/docs/zero_to_hero_guide/Tool_Calling101_Using_Together's_Llama_Stack_Server.ipynb +++ b/docs/zero_to_hero_guide/Tool_Calling101_Using_Together's_Llama_Stack_Server.ipynb @@ -83,8 +83,8 @@ }, "outputs": [], "source": [ - "LLAMA_STACK_API_TOGETHER_URL=\"https://llama-stack.together.ai\"\n", - "LLAMA31_8B_INSTRUCT = \"Llama3.1-8B-Instruct\"" + "LLAMA_STACK_API_TOGETHER_URL = \"https://llama-stack.together.ai\"\n", + "LLAMA31_8B_INSTRUCT = \"Llama3.1-8B-Instruct\"\n" ] }, { @@ -107,12 +107,13 @@ " AgentConfigToolSearchToolDefinition,\n", ")\n", "\n", + "\n", "# Helper function to create an agent with tools\n", "async def create_tool_agent(\n", " client: LlamaStackClient,\n", " tools: List[Dict],\n", " instructions: str = \"You are a helpful assistant\",\n", - " model: str = LLAMA31_8B_INSTRUCT\n", + " model: str = LLAMA31_8B_INSTRUCT,\n", ") -> Agent:\n", " \"\"\"Create an agent with specified tools.\"\"\"\n", " print(\"Using the following model: \", model)\n", @@ -120,9 +121,9 @@ " model=model,\n", " instructions=instructions,\n", " sampling_params={\n", - " \"strategy\": \"greedy\",\n", - " \"temperature\": 1.0,\n", - " \"top_p\": 0.9,\n", + " \"strategy\": {\n", + " \"type\": \"greedy\",\n", + " },\n", " },\n", " tools=tools,\n", " tool_choice=\"auto\",\n", @@ -130,7 +131,7 @@ " enable_session_persistence=True,\n", " )\n", "\n", - " return Agent(client, agent_config)" + " return Agent(client, agent_config)\n" ] }, { @@ -172,7 +173,8 @@ ], "source": [ "# comment this if you don't have a BRAVE_SEARCH_API_KEY\n", - "os.environ[\"BRAVE_SEARCH_API_KEY\"] = 'YOUR_BRAVE_SEARCH_API_KEY'\n", + "os.environ[\"BRAVE_SEARCH_API_KEY\"] = \"YOUR_BRAVE_SEARCH_API_KEY\"\n", + "\n", "\n", "async def create_search_agent(client: LlamaStackClient) -> Agent:\n", " \"\"\"Create an agent with Brave Search capability.\"\"\"\n", @@ -186,8 +188,8 @@ "\n", " return await create_tool_agent(\n", " client=client,\n", - " tools=[search_tool], # set this to [] if you don't have a BRAVE_SEARCH_API_KEY\n", - " model = LLAMA31_8B_INSTRUCT,\n", + " tools=[search_tool], # set this to [] if you don't have a BRAVE_SEARCH_API_KEY\n", + " model=LLAMA31_8B_INSTRUCT,\n", " instructions=\"\"\"\n", " You are a research assistant that can search the web.\n", " Always cite your sources with URLs when providing information.\n", @@ -198,9 +200,10 @@ "\n", " SOURCES:\n", " - [Source title](URL)\n", - " \"\"\"\n", + " \"\"\",\n", " )\n", "\n", + "\n", "# Example usage\n", "async def search_example():\n", " client = LlamaStackClient(base_url=LLAMA_STACK_API_TOGETHER_URL)\n", @@ -212,7 +215,7 @@ " # Example queries\n", " queries = [\n", " \"What are the latest developments in quantum computing?\",\n", - " #\"Who won the most recent Super Bowl?\",\n", + " # \"Who won the most recent Super Bowl?\",\n", " ]\n", "\n", " for query in queries:\n", @@ -227,8 +230,9 @@ " async for log in EventLogger().log(response):\n", " log.print()\n", "\n", + "\n", "# Run the example (in Jupyter, use asyncio.run())\n", - "await search_example()" + "await search_example()\n" ] }, { @@ -286,12 +290,16 @@ } ], "source": [ - "from typing import TypedDict, Optional, Dict, Any\n", - "from datetime import datetime\n", "import json\n", - "from llama_stack_client.types.tool_param_definition_param import ToolParamDefinitionParam\n", - "from llama_stack_client.types import CompletionMessage,ToolResponseMessage\n", + "from datetime import datetime\n", + "from typing import Any, Dict, Optional, TypedDict\n", + "\n", "from llama_stack_client.lib.agents.custom_tool import CustomTool\n", + "from llama_stack_client.types import CompletionMessage, ToolResponseMessage\n", + "from llama_stack_client.types.tool_param_definition_param import (\n", + " ToolParamDefinitionParam,\n", + ")\n", + "\n", "\n", "class WeatherTool(CustomTool):\n", " \"\"\"Example custom tool for weather information.\"\"\"\n", @@ -305,16 +313,15 @@ " def get_params_definition(self) -> Dict[str, ToolParamDefinitionParam]:\n", " return {\n", " \"location\": ToolParamDefinitionParam(\n", - " param_type=\"str\",\n", - " description=\"City or location name\",\n", - " required=True\n", + " param_type=\"str\", description=\"City or location name\", required=True\n", " ),\n", " \"date\": ToolParamDefinitionParam(\n", " param_type=\"str\",\n", " description=\"Optional date (YYYY-MM-DD)\",\n", - " required=False\n", - " )\n", + " required=False,\n", + " ),\n", " }\n", + "\n", " async def run(self, messages: List[CompletionMessage]) -> List[ToolResponseMessage]:\n", " assert len(messages) == 1, \"Expected single message\"\n", "\n", @@ -337,20 +344,14 @@ " )\n", " return [message]\n", "\n", - " async def run_impl(self, location: str, date: Optional[str] = None) -> Dict[str, Any]:\n", + " async def run_impl(\n", + " self, location: str, date: Optional[str] = None\n", + " ) -> Dict[str, Any]:\n", " \"\"\"Simulate getting weather data (replace with actual API call).\"\"\"\n", " # Mock implementation\n", " if date:\n", - " return {\n", - " \"temperature\": 90.1,\n", - " \"conditions\": \"sunny\",\n", - " \"humidity\": 40.0\n", - " }\n", - " return {\n", - " \"temperature\": 72.5,\n", - " \"conditions\": \"partly cloudy\",\n", - " \"humidity\": 65.0\n", - " }\n", + " return {\"temperature\": 90.1, \"conditions\": \"sunny\", \"humidity\": 40.0}\n", + " return {\"temperature\": 72.5, \"conditions\": \"partly cloudy\", \"humidity\": 65.0}\n", "\n", "\n", "async def create_weather_agent(client: LlamaStackClient) -> Agent:\n", @@ -358,38 +359,33 @@ "\n", " # Create the agent with the tool\n", " weather_tool = WeatherTool()\n", - " \n", + "\n", " agent_config = AgentConfig(\n", " model=LLAMA31_8B_INSTRUCT,\n", - " #model=model_name,\n", + " # model=model_name,\n", " instructions=\"\"\"\n", " You are a weather assistant that can provide weather information.\n", " Always specify the location clearly in your responses.\n", " Include both temperature and conditions in your summaries.\n", " \"\"\",\n", " sampling_params={\n", - " \"strategy\": \"greedy\",\n", - " \"temperature\": 1.0,\n", - " \"top_p\": 0.9,\n", + " \"strategy\": {\n", + " \"type\": \"greedy\",\n", + " },\n", " },\n", - " tools=[\n", - " weather_tool.get_tool_definition()\n", - " ],\n", + " tools=[weather_tool.get_tool_definition()],\n", " tool_choice=\"auto\",\n", " tool_prompt_format=\"json\",\n", " input_shields=[],\n", " output_shields=[],\n", - " enable_session_persistence=True\n", + " enable_session_persistence=True,\n", " )\n", "\n", - " agent = Agent(\n", - " client=client,\n", - " agent_config=agent_config,\n", - " custom_tools=[weather_tool]\n", - " )\n", + " agent = Agent(client=client, agent_config=agent_config, custom_tools=[weather_tool])\n", "\n", " return agent\n", "\n", + "\n", "# Example usage\n", "async def weather_example():\n", " client = LlamaStackClient(base_url=LLAMA_STACK_API_TOGETHER_URL)\n", @@ -413,12 +409,14 @@ " async for log in EventLogger().log(response):\n", " log.print()\n", "\n", + "\n", "# For Jupyter notebooks\n", "import nest_asyncio\n", + "\n", "nest_asyncio.apply()\n", "\n", "# Run the example\n", - "await weather_example()" + "await weather_example()\n" ] }, { diff --git a/llama_stack/apis/agents/agents.py b/llama_stack/apis/agents/agents.py index c3f3d21f0..c19f28054 100644 --- a/llama_stack/apis/agents/agents.py +++ b/llama_stack/apis/agents/agents.py @@ -7,6 +7,7 @@ from datetime import datetime from enum import Enum from typing import ( + Annotated, Any, AsyncIterator, Dict, @@ -20,7 +21,6 @@ from typing import ( from llama_models.schema_utils import json_schema_type, register_schema, webmethod from pydantic import BaseModel, ConfigDict, Field -from typing_extensions import Annotated from llama_stack.apis.common.content_types import ContentDelta, InterleavedContent, URL from llama_stack.apis.inference import ( @@ -33,7 +33,6 @@ from llama_stack.apis.inference import ( ToolResponseMessage, UserMessage, ) -from llama_stack.apis.memory import MemoryBank from llama_stack.apis.safety import SafetyViolation from llama_stack.apis.tools import ToolDef from llama_stack.providers.utils.telemetry.trace_protocol import trace_protocol @@ -89,7 +88,7 @@ class MemoryRetrievalStep(StepCommon): step_type: Literal[StepType.memory_retrieval.value] = ( StepType.memory_retrieval.value ) - memory_bank_ids: List[str] + vector_db_ids: str inserted_context: InterleavedContent @@ -133,8 +132,6 @@ class Session(BaseModel): turns: List[Turn] started_at: datetime - memory_bank: Optional[MemoryBank] = None - class AgentToolGroupWithArgs(BaseModel): name: str @@ -296,13 +293,13 @@ class AgentStepResponse(BaseModel): @runtime_checkable @trace_protocol class Agents(Protocol): - @webmethod(route="/agents/create") + @webmethod(route="/agents", method="POST") async def create_agent( self, agent_config: AgentConfig, ) -> AgentCreateResponse: ... - @webmethod(route="/agents/turn/create") + @webmethod(route="/agents/{agent_id}/session/{session_id}/turn", method="POST") async def create_agent_turn( self, agent_id: str, @@ -318,36 +315,52 @@ class Agents(Protocol): toolgroups: Optional[List[AgentToolGroup]] = None, ) -> Union[Turn, AsyncIterator[AgentTurnResponseStreamChunk]]: ... - @webmethod(route="/agents/turn/get") + @webmethod( + route="/agents/{agent_id}/session/{session_id}/turn/{turn_id}", method="GET" + ) async def get_agents_turn( - self, agent_id: str, session_id: str, turn_id: str + self, + agent_id: str, + session_id: str, + turn_id: str, ) -> Turn: ... - @webmethod(route="/agents/step/get") + @webmethod( + route="/agents/{agent_id}/session/{session_id}/turn/{turn_id}/step/{step_id}", + method="GET", + ) async def get_agents_step( - self, agent_id: str, session_id: str, turn_id: str, step_id: str + self, + agent_id: str, + session_id: str, + turn_id: str, + step_id: str, ) -> AgentStepResponse: ... - @webmethod(route="/agents/session/create") + @webmethod(route="/agents/{agent_id}/session", method="POST") async def create_agent_session( self, agent_id: str, session_name: str, ) -> AgentSessionCreateResponse: ... - @webmethod(route="/agents/session/get") + @webmethod(route="/agents/{agent_id}/session/{session_id}", method="GET") async def get_agents_session( self, - agent_id: str, session_id: str, + agent_id: str, turn_ids: Optional[List[str]] = None, ) -> Session: ... - @webmethod(route="/agents/session/delete") - async def delete_agents_session(self, agent_id: str, session_id: str) -> None: ... + @webmethod(route="/agents/{agent_id}/session/{session_id}", method="DELETE") + async def delete_agents_session( + self, + session_id: str, + agent_id: str, + ) -> None: ... - @webmethod(route="/agents/delete") - async def delete_agents( + @webmethod(route="/agents/{agent_id}", method="DELETE") + async def delete_agent( self, agent_id: str, ) -> None: ... diff --git a/llama_stack/apis/agents/event_logger.py b/llama_stack/apis/agents/event_logger.py index 9e2f14805..7a607ffda 100644 --- a/llama_stack/apis/agents/event_logger.py +++ b/llama_stack/apis/agents/event_logger.py @@ -137,7 +137,7 @@ class EventLogger: event, LogEvent( role=None, - content=delta.content, + content=delta.tool_call, end="", color="cyan", ), @@ -208,7 +208,7 @@ class EventLogger: ): details = event.payload.step_details inserted_context = interleaved_content_as_str(details.inserted_context) - content = f"fetched {len(inserted_context)} bytes from {details.memory_bank_ids}" + content = f"fetched {len(inserted_context)} bytes from {details.vector_db_ids}" yield ( event, diff --git a/llama_stack/apis/batch_inference/batch_inference.py b/llama_stack/apis/batch_inference/batch_inference.py index 81826a7b1..ca5ba059f 100644 --- a/llama_stack/apis/batch_inference/batch_inference.py +++ b/llama_stack/apis/batch_inference/batch_inference.py @@ -54,7 +54,7 @@ class BatchChatCompletionResponse(BaseModel): @runtime_checkable class BatchInference(Protocol): - @webmethod(route="/batch-inference/completion") + @webmethod(route="/batch-inference/completion", method="POST") async def batch_completion( self, model: str, @@ -63,7 +63,7 @@ class BatchInference(Protocol): logprobs: Optional[LogProbConfig] = None, ) -> BatchCompletionResponse: ... - @webmethod(route="/batch-inference/chat-completion") + @webmethod(route="/batch-inference/chat-completion", method="POST") async def batch_chat_completion( self, model: str, diff --git a/llama_stack/apis/common/content_types.py b/llama_stack/apis/common/content_types.py index 3b61fa243..1d8cea567 100644 --- a/llama_stack/apis/common/content_types.py +++ b/llama_stack/apis/common/content_types.py @@ -38,8 +38,9 @@ class _URLOrData(BaseModel): @json_schema_type -class ImageContentItem(_URLOrData): +class ImageContentItem(BaseModel): type: Literal["image"] = "image" + image: _URLOrData @json_schema_type @@ -64,14 +65,16 @@ InterleavedContent = register_schema( ) +@json_schema_type class TextDelta(BaseModel): type: Literal["text"] = "text" text: str +@json_schema_type class ImageDelta(BaseModel): type: Literal["image"] = "image" - data: bytes + image: bytes @json_schema_type @@ -89,7 +92,7 @@ class ToolCallDelta(BaseModel): # you either send an in-progress tool call so the client can stream a long # code generation or you send the final parsed tool call at the end of the # stream - content: Union[str, ToolCall] + tool_call: Union[str, ToolCall] parse_status: ToolCallParseStatus diff --git a/llama_stack/apis/common/type_system.py b/llama_stack/apis/common/type_system.py index a653efef9..fa9c5e92e 100644 --- a/llama_stack/apis/common/type_system.py +++ b/llama_stack/apis/common/type_system.py @@ -6,54 +6,71 @@ from typing import Literal, Union -from llama_models.schema_utils import register_schema +from llama_models.schema_utils import json_schema_type, register_schema from pydantic import BaseModel, Field from typing_extensions import Annotated +@json_schema_type class StringType(BaseModel): type: Literal["string"] = "string" +@json_schema_type class NumberType(BaseModel): type: Literal["number"] = "number" +@json_schema_type class BooleanType(BaseModel): type: Literal["boolean"] = "boolean" +@json_schema_type class ArrayType(BaseModel): type: Literal["array"] = "array" +@json_schema_type class ObjectType(BaseModel): type: Literal["object"] = "object" +@json_schema_type class JsonType(BaseModel): type: Literal["json"] = "json" +@json_schema_type class UnionType(BaseModel): type: Literal["union"] = "union" +@json_schema_type class ChatCompletionInputType(BaseModel): # expects List[Message] for messages type: Literal["chat_completion_input"] = "chat_completion_input" +@json_schema_type class CompletionInputType(BaseModel): # expects InterleavedTextMedia for content type: Literal["completion_input"] = "completion_input" +@json_schema_type class AgentTurnInputType(BaseModel): # expects List[Message] for messages (may also include attachments?) type: Literal["agent_turn_input"] = "agent_turn_input" +@json_schema_type +class DialogType(BaseModel): + # expects List[Message] for messages + # this type semantically contains the output label whereas ChatCompletionInputType does not + type: Literal["dialog"] = "dialog" + + ParamType = register_schema( Annotated[ Union[ diff --git a/llama_stack/apis/datasetio/datasetio.py b/llama_stack/apis/datasetio/datasetio.py index 983e0e4ea..8b4c25a1d 100644 --- a/llama_stack/apis/datasetio/datasetio.py +++ b/llama_stack/apis/datasetio/datasetio.py @@ -29,7 +29,7 @@ class DatasetIO(Protocol): # keeping for aligning with inference/safety, but this is not used dataset_store: DatasetStore - @webmethod(route="/datasetio/get-rows-paginated", method="GET") + @webmethod(route="/datasetio/rows", method="GET") async def get_rows_paginated( self, dataset_id: str, @@ -38,7 +38,7 @@ class DatasetIO(Protocol): filter_condition: Optional[str] = None, ) -> PaginatedRowsResult: ... - @webmethod(route="/datasetio/append-rows", method="POST") + @webmethod(route="/datasetio/rows", method="POST") async def append_rows( self, dataset_id: str, rows: List[Dict[str, Any]] ) -> None: ... diff --git a/llama_stack/apis/datasets/datasets.py b/llama_stack/apis/datasets/datasets.py index 7afc0f8fd..5ad5bdcdb 100644 --- a/llama_stack/apis/datasets/datasets.py +++ b/llama_stack/apis/datasets/datasets.py @@ -7,11 +7,9 @@ from typing import Any, Dict, List, Literal, Optional, Protocol from llama_models.schema_utils import json_schema_type, webmethod - from pydantic import BaseModel, Field from llama_stack.apis.common.content_types import URL - from llama_stack.apis.common.type_system import ParamType from llama_stack.apis.resource import Resource, ResourceType @@ -44,8 +42,12 @@ class DatasetInput(CommonDatasetFields, BaseModel): provider_dataset_id: Optional[str] = None +class ListDatasetsResponse(BaseModel): + data: List[Dataset] + + class Datasets(Protocol): - @webmethod(route="/datasets/register", method="POST") + @webmethod(route="/datasets", method="POST") async def register_dataset( self, dataset_id: str, @@ -56,16 +58,16 @@ class Datasets(Protocol): metadata: Optional[Dict[str, Any]] = None, ) -> None: ... - @webmethod(route="/datasets/get", method="GET") + @webmethod(route="/datasets/{dataset_id}", method="GET") async def get_dataset( self, dataset_id: str, ) -> Optional[Dataset]: ... - @webmethod(route="/datasets/list", method="GET") - async def list_datasets(self) -> List[Dataset]: ... + @webmethod(route="/datasets", method="GET") + async def list_datasets(self) -> ListDatasetsResponse: ... - @webmethod(route="/datasets/unregister", method="POST") + @webmethod(route="/datasets/{dataset_id}", method="DELETE") async def unregister_dataset( self, dataset_id: str, diff --git a/llama_stack/apis/datatypes.py b/llama_stack/apis/datatypes.py new file mode 100644 index 000000000..ccc395b80 --- /dev/null +++ b/llama_stack/apis/datatypes.py @@ -0,0 +1,35 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +from enum import Enum + +from llama_models.schema_utils import json_schema_type + + +@json_schema_type +class Api(Enum): + inference = "inference" + safety = "safety" + agents = "agents" + vector_io = "vector_io" + datasetio = "datasetio" + scoring = "scoring" + eval = "eval" + post_training = "post_training" + tool_runtime = "tool_runtime" + + telemetry = "telemetry" + + models = "models" + shields = "shields" + vector_dbs = "vector_dbs" + datasets = "datasets" + scoring_functions = "scoring_functions" + eval_tasks = "eval_tasks" + tool_groups = "tool_groups" + + # built-in API + inspect = "inspect" diff --git a/llama_stack/apis/eval/eval.py b/llama_stack/apis/eval/eval.py index 1073d6310..c9d2fb70b 100644 --- a/llama_stack/apis/eval/eval.py +++ b/llama_stack/apis/eval/eval.py @@ -7,9 +7,7 @@ from typing import Any, Dict, List, Literal, Optional, Protocol, Union from llama_models.schema_utils import json_schema_type, webmethod - from pydantic import BaseModel, Field - from typing_extensions import Annotated from llama_stack.apis.agents import AgentConfig @@ -76,14 +74,14 @@ class EvaluateResponse(BaseModel): class Eval(Protocol): - @webmethod(route="/eval/run-eval", method="POST") + @webmethod(route="/eval/tasks/{task_id}/jobs", method="POST") async def run_eval( self, task_id: str, task_config: EvalTaskConfig, ) -> Job: ... - @webmethod(route="/eval/evaluate-rows", method="POST") + @webmethod(route="/eval/tasks/{task_id}/evaluations", method="POST") async def evaluate_rows( self, task_id: str, @@ -92,11 +90,11 @@ class Eval(Protocol): task_config: EvalTaskConfig, ) -> EvaluateResponse: ... - @webmethod(route="/eval/job/status", method="GET") + @webmethod(route="/eval/tasks/{task_id}/jobs/{job_id}", method="GET") async def job_status(self, task_id: str, job_id: str) -> Optional[JobStatus]: ... - @webmethod(route="/eval/job/cancel", method="POST") + @webmethod(route="/eval/tasks/{task_id}/jobs/{job_id}", method="DELETE") async def job_cancel(self, task_id: str, job_id: str) -> None: ... - @webmethod(route="/eval/job/result", method="GET") - async def job_result(self, task_id: str, job_id: str) -> EvaluateResponse: ... + @webmethod(route="/eval/tasks/{task_id}/jobs/{job_id}/result", method="GET") + async def job_result(self, job_id: str, task_id: str) -> EvaluateResponse: ... diff --git a/llama_stack/apis/eval_tasks/eval_tasks.py b/llama_stack/apis/eval_tasks/eval_tasks.py index 083681289..a0a533055 100644 --- a/llama_stack/apis/eval_tasks/eval_tasks.py +++ b/llama_stack/apis/eval_tasks/eval_tasks.py @@ -6,7 +6,6 @@ from typing import Any, Dict, List, Literal, Optional, Protocol, runtime_checkable from llama_models.schema_utils import json_schema_type, webmethod - from pydantic import BaseModel, Field from llama_stack.apis.resource import Resource, ResourceType @@ -40,15 +39,22 @@ class EvalTaskInput(CommonEvalTaskFields, BaseModel): provider_eval_task_id: Optional[str] = None +class ListEvalTasksResponse(BaseModel): + data: List[EvalTask] + + @runtime_checkable class EvalTasks(Protocol): - @webmethod(route="/eval-tasks/list", method="GET") - async def list_eval_tasks(self) -> List[EvalTask]: ... + @webmethod(route="/eval-tasks", method="GET") + async def list_eval_tasks(self) -> ListEvalTasksResponse: ... - @webmethod(route="/eval-tasks/get", method="GET") - async def get_eval_task(self, name: str) -> Optional[EvalTask]: ... + @webmethod(route="/eval-tasks/{eval_task_id}", method="GET") + async def get_eval_task( + self, + eval_task_id: str, + ) -> Optional[EvalTask]: ... - @webmethod(route="/eval-tasks/register", method="POST") + @webmethod(route="/eval-tasks", method="POST") async def register_eval_task( self, eval_task_id: str, diff --git a/llama_stack/apis/inference/inference.py b/llama_stack/apis/inference/inference.py index b525aa331..fdda5fe1b 100644 --- a/llama_stack/apis/inference/inference.py +++ b/llama_stack/apis/inference/inference.py @@ -291,7 +291,7 @@ class ModelStore(Protocol): class Inference(Protocol): model_store: ModelStore - @webmethod(route="/inference/completion") + @webmethod(route="/inference/completion", method="POST") async def completion( self, model_id: str, @@ -302,7 +302,7 @@ class Inference(Protocol): logprobs: Optional[LogProbConfig] = None, ) -> Union[CompletionResponse, AsyncIterator[CompletionResponseStreamChunk]]: ... - @webmethod(route="/inference/chat-completion") + @webmethod(route="/inference/chat-completion", method="POST") async def chat_completion( self, model_id: str, @@ -319,7 +319,7 @@ class Inference(Protocol): ChatCompletionResponse, AsyncIterator[ChatCompletionResponseStreamChunk] ]: ... - @webmethod(route="/inference/embeddings") + @webmethod(route="/inference/embeddings", method="POST") async def embeddings( self, model_id: str, diff --git a/llama_stack/apis/inspect/inspect.py b/llama_stack/apis/inspect/inspect.py index 699bce7b7..cd51469c1 100644 --- a/llama_stack/apis/inspect/inspect.py +++ b/llama_stack/apis/inspect/inspect.py @@ -4,7 +4,7 @@ # This source code is licensed under the terms described in the LICENSE file in # the root directory of this source tree. -from typing import Dict, List, Protocol, runtime_checkable +from typing import List, Protocol, runtime_checkable from llama_models.schema_utils import json_schema_type, webmethod from pydantic import BaseModel @@ -12,6 +12,7 @@ from pydantic import BaseModel @json_schema_type class ProviderInfo(BaseModel): + api: str provider_id: str provider_type: str @@ -34,13 +35,21 @@ class VersionInfo(BaseModel): version: str +class ListProvidersResponse(BaseModel): + data: List[ProviderInfo] + + +class ListRoutesResponse(BaseModel): + data: List[RouteInfo] + + @runtime_checkable class Inspect(Protocol): - @webmethod(route="/providers/list", method="GET") - async def list_providers(self) -> Dict[str, ProviderInfo]: ... + @webmethod(route="/inspect/providers", method="GET") + async def list_providers(self) -> ListProvidersResponse: ... - @webmethod(route="/routes/list", method="GET") - async def list_routes(self) -> Dict[str, List[RouteInfo]]: ... + @webmethod(route="/inspect/routes", method="GET") + async def list_routes(self) -> ListRoutesResponse: ... @webmethod(route="/health", method="GET") async def health(self) -> HealthInfo: ... diff --git a/llama_stack/apis/memory_banks/memory_banks.py b/llama_stack/apis/memory_banks/memory_banks.py deleted file mode 100644 index b037dfa66..000000000 --- a/llama_stack/apis/memory_banks/memory_banks.py +++ /dev/null @@ -1,152 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the terms described in the LICENSE file in -# the root directory of this source tree. - -from enum import Enum -from typing import ( - Annotated, - List, - Literal, - Optional, - Protocol, - runtime_checkable, - Union, -) - -from llama_models.schema_utils import json_schema_type, webmethod - -from pydantic import BaseModel, Field - -from llama_stack.apis.resource import Resource, ResourceType -from llama_stack.providers.utils.telemetry.trace_protocol import trace_protocol - - -@json_schema_type -class MemoryBankType(Enum): - vector = "vector" - keyvalue = "keyvalue" - keyword = "keyword" - graph = "graph" - - -# define params for each type of memory bank, this leads to a tagged union -# accepted as input from the API or from the config. -@json_schema_type -class VectorMemoryBankParams(BaseModel): - memory_bank_type: Literal[MemoryBankType.vector.value] = MemoryBankType.vector.value - embedding_model: str - chunk_size_in_tokens: int - overlap_size_in_tokens: Optional[int] = None - - -@json_schema_type -class KeyValueMemoryBankParams(BaseModel): - memory_bank_type: Literal[MemoryBankType.keyvalue.value] = ( - MemoryBankType.keyvalue.value - ) - - -@json_schema_type -class KeywordMemoryBankParams(BaseModel): - memory_bank_type: Literal[MemoryBankType.keyword.value] = ( - MemoryBankType.keyword.value - ) - - -@json_schema_type -class GraphMemoryBankParams(BaseModel): - memory_bank_type: Literal[MemoryBankType.graph.value] = MemoryBankType.graph.value - - -BankParams = Annotated[ - Union[ - VectorMemoryBankParams, - KeyValueMemoryBankParams, - KeywordMemoryBankParams, - GraphMemoryBankParams, - ], - Field(discriminator="memory_bank_type"), -] - - -# Some common functionality for memory banks. -class MemoryBankResourceMixin(Resource): - type: Literal[ResourceType.memory_bank.value] = ResourceType.memory_bank.value - - @property - def memory_bank_id(self) -> str: - return self.identifier - - @property - def provider_memory_bank_id(self) -> str: - return self.provider_resource_id - - -@json_schema_type -class VectorMemoryBank(MemoryBankResourceMixin): - memory_bank_type: Literal[MemoryBankType.vector.value] = MemoryBankType.vector.value - embedding_model: str - chunk_size_in_tokens: int - embedding_dimension: Optional[int] = 384 # default to minilm-l6-v2 - overlap_size_in_tokens: Optional[int] = None - - -@json_schema_type -class KeyValueMemoryBank(MemoryBankResourceMixin): - memory_bank_type: Literal[MemoryBankType.keyvalue.value] = ( - MemoryBankType.keyvalue.value - ) - - -# TODO: KeyValue and Keyword are so similar in name, oof. Get a better naming convention. -@json_schema_type -class KeywordMemoryBank(MemoryBankResourceMixin): - memory_bank_type: Literal[MemoryBankType.keyword.value] = ( - MemoryBankType.keyword.value - ) - - -@json_schema_type -class GraphMemoryBank(MemoryBankResourceMixin): - memory_bank_type: Literal[MemoryBankType.graph.value] = MemoryBankType.graph.value - - -MemoryBank = Annotated[ - Union[ - VectorMemoryBank, - KeyValueMemoryBank, - KeywordMemoryBank, - GraphMemoryBank, - ], - Field(discriminator="memory_bank_type"), -] - - -class MemoryBankInput(BaseModel): - memory_bank_id: str - params: BankParams - provider_memory_bank_id: Optional[str] = None - - -@runtime_checkable -@trace_protocol -class MemoryBanks(Protocol): - @webmethod(route="/memory-banks/list", method="GET") - async def list_memory_banks(self) -> List[MemoryBank]: ... - - @webmethod(route="/memory-banks/get", method="GET") - async def get_memory_bank(self, memory_bank_id: str) -> Optional[MemoryBank]: ... - - @webmethod(route="/memory-banks/register", method="POST") - async def register_memory_bank( - self, - memory_bank_id: str, - params: BankParams, - provider_id: Optional[str] = None, - provider_memory_bank_id: Optional[str] = None, - ) -> MemoryBank: ... - - @webmethod(route="/memory-banks/unregister", method="POST") - async def unregister_memory_bank(self, memory_bank_id: str) -> None: ... diff --git a/llama_stack/apis/models/models.py b/llama_stack/apis/models/models.py index 0ee23ecc1..3361c2836 100644 --- a/llama_stack/apis/models/models.py +++ b/llama_stack/apis/models/models.py @@ -52,16 +52,23 @@ class ModelInput(CommonModelFields): model_config = ConfigDict(protected_namespaces=()) +class ListModelsResponse(BaseModel): + data: List[Model] + + @runtime_checkable @trace_protocol class Models(Protocol): - @webmethod(route="/models/list", method="GET") - async def list_models(self) -> List[Model]: ... + @webmethod(route="/models", method="GET") + async def list_models(self) -> ListModelsResponse: ... - @webmethod(route="/models/get", method="GET") - async def get_model(self, identifier: str) -> Optional[Model]: ... + @webmethod(route="/models/{model_id}", method="GET") + async def get_model( + self, + model_id: str, + ) -> Optional[Model]: ... - @webmethod(route="/models/register", method="POST") + @webmethod(route="/models", method="POST") async def register_model( self, model_id: str, @@ -71,5 +78,8 @@ class Models(Protocol): model_type: Optional[ModelType] = None, ) -> Model: ... - @webmethod(route="/models/unregister", method="POST") - async def unregister_model(self, model_id: str) -> None: ... + @webmethod(route="/models/{model_id}", method="DELETE") + async def unregister_model( + self, + model_id: str, + ) -> None: ... diff --git a/llama_stack/apis/post_training/post_training.py b/llama_stack/apis/post_training/post_training.py index 8e1edbe87..b9aa3bbde 100644 --- a/llama_stack/apis/post_training/post_training.py +++ b/llama_stack/apis/post_training/post_training.py @@ -6,16 +6,13 @@ from datetime import datetime from enum import Enum - from typing import Any, Dict, List, Literal, Optional, Protocol, Union from llama_models.schema_utils import json_schema_type, webmethod - from pydantic import BaseModel, Field from typing_extensions import Annotated from llama_stack.apis.common.content_types import URL - from llama_stack.apis.common.job_types import JobStatus from llama_stack.apis.common.training_types import Checkpoint @@ -27,11 +24,18 @@ class OptimizerType(Enum): sgd = "sgd" +@json_schema_type +class DatasetFormat(Enum): + instruct = "instruct" + dialog = "dialog" + + @json_schema_type class DataConfig(BaseModel): dataset_id: str batch_size: int shuffle: bool + data_format: DatasetFormat validation_dataset_id: Optional[str] = None packed: Optional[bool] = False train_on_input: Optional[bool] = False @@ -152,6 +156,10 @@ class PostTrainingJobStatusResponse(BaseModel): checkpoints: List[Checkpoint] = Field(default_factory=list) +class ListPostTrainingJobsResponse(BaseModel): + data: List[PostTrainingJob] + + @json_schema_type class PostTrainingJobArtifactsResponse(BaseModel): """Artifacts of a finetuning job.""" @@ -190,7 +198,7 @@ class PostTraining(Protocol): ) -> PostTrainingJob: ... @webmethod(route="/post-training/jobs", method="GET") - async def get_training_jobs(self) -> List[PostTrainingJob]: ... + async def get_training_jobs(self) -> ListPostTrainingJobsResponse: ... @webmethod(route="/post-training/job/status", method="GET") async def get_training_job_status( diff --git a/llama_stack/apis/resource.py b/llama_stack/apis/resource.py index a85f5a31c..d0ce72644 100644 --- a/llama_stack/apis/resource.py +++ b/llama_stack/apis/resource.py @@ -14,7 +14,7 @@ from pydantic import BaseModel, Field class ResourceType(Enum): model = "model" shield = "shield" - memory_bank = "memory_bank" + vector_db = "vector_db" dataset = "dataset" scoring_function = "scoring_function" eval_task = "eval_task" @@ -37,5 +37,5 @@ class Resource(BaseModel): provider_id: str = Field(description="ID of the provider that owns this resource") type: ResourceType = Field( - description="Type of resource (e.g. 'model', 'shield', 'memory_bank', etc.)" + description="Type of resource (e.g. 'model', 'shield', 'vector_db', etc.)" ) diff --git a/llama_stack/apis/safety/safety.py b/llama_stack/apis/safety/safety.py index dd24642b1..513733d1e 100644 --- a/llama_stack/apis/safety/safety.py +++ b/llama_stack/apis/safety/safety.py @@ -12,7 +12,6 @@ from pydantic import BaseModel, Field from llama_stack.apis.inference import Message from llama_stack.apis.shields import Shield - from llama_stack.providers.utils.telemetry.trace_protocol import trace_protocol @@ -49,7 +48,7 @@ class ShieldStore(Protocol): class Safety(Protocol): shield_store: ShieldStore - @webmethod(route="/safety/run-shield") + @webmethod(route="/safety/run-shield", method="POST") async def run_shield( self, shield_id: str, diff --git a/llama_stack/apis/scoring/scoring.py b/llama_stack/apis/scoring/scoring.py index 996291dcc..5bacaaf66 100644 --- a/llama_stack/apis/scoring/scoring.py +++ b/llama_stack/apis/scoring/scoring.py @@ -11,7 +11,6 @@ from pydantic import BaseModel from llama_stack.apis.scoring_functions import ScoringFn, ScoringFnParams - # mapping of metric to value ScoringResultRow = Dict[str, Any] @@ -43,7 +42,7 @@ class ScoringFunctionStore(Protocol): class Scoring(Protocol): scoring_function_store: ScoringFunctionStore - @webmethod(route="/scoring/score-batch") + @webmethod(route="/scoring/score-batch", method="POST") async def score_batch( self, dataset_id: str, @@ -51,7 +50,7 @@ class Scoring(Protocol): save_results_dataset: bool = False, ) -> ScoreBatchResponse: ... - @webmethod(route="/scoring/score") + @webmethod(route="/scoring/score", method="POST") async def score( self, input_rows: List[Dict[str, Any]], diff --git a/llama_stack/apis/scoring_functions/scoring_functions.py b/llama_stack/apis/scoring_functions/scoring_functions.py index fc57cfbbf..3089dc0a4 100644 --- a/llama_stack/apis/scoring_functions/scoring_functions.py +++ b/llama_stack/apis/scoring_functions/scoring_functions.py @@ -21,7 +21,6 @@ from pydantic import BaseModel, Field from typing_extensions import Annotated from llama_stack.apis.common.type_system import ParamType - from llama_stack.apis.resource import Resource, ResourceType @@ -129,15 +128,21 @@ class ScoringFnInput(CommonScoringFnFields, BaseModel): provider_scoring_fn_id: Optional[str] = None +class ListScoringFunctionsResponse(BaseModel): + data: List[ScoringFn] + + @runtime_checkable class ScoringFunctions(Protocol): - @webmethod(route="/scoring-functions/list", method="GET") - async def list_scoring_functions(self) -> List[ScoringFn]: ... + @webmethod(route="/scoring-functions", method="GET") + async def list_scoring_functions(self) -> ListScoringFunctionsResponse: ... - @webmethod(route="/scoring-functions/get", method="GET") - async def get_scoring_function(self, scoring_fn_id: str) -> Optional[ScoringFn]: ... + @webmethod(route="/scoring-functions/{scoring_fn_id}", method="GET") + async def get_scoring_function( + self, scoring_fn_id: str, / + ) -> Optional[ScoringFn]: ... - @webmethod(route="/scoring-functions/register", method="POST") + @webmethod(route="/scoring-functions", method="POST") async def register_scoring_function( self, scoring_fn_id: str, diff --git a/llama_stack/apis/shields/shields.py b/llama_stack/apis/shields/shields.py index 8d4d5f9fd..3dd685b14 100644 --- a/llama_stack/apis/shields/shields.py +++ b/llama_stack/apis/shields/shields.py @@ -38,16 +38,20 @@ class ShieldInput(CommonShieldFields): provider_shield_id: Optional[str] = None +class ListShieldsResponse(BaseModel): + data: List[Shield] + + @runtime_checkable @trace_protocol class Shields(Protocol): - @webmethod(route="/shields/list", method="GET") - async def list_shields(self) -> List[Shield]: ... + @webmethod(route="/shields", method="GET") + async def list_shields(self) -> ListShieldsResponse: ... - @webmethod(route="/shields/get", method="GET") + @webmethod(route="/shields/{identifier}", method="GET") async def get_shield(self, identifier: str) -> Optional[Shield]: ... - @webmethod(route="/shields/register", method="POST") + @webmethod(route="/shields", method="POST") async def register_shield( self, shield_id: str, diff --git a/llama_stack/apis/telemetry/telemetry.py b/llama_stack/apis/telemetry/telemetry.py index 23a475bff..30a4e2342 100644 --- a/llama_stack/apis/telemetry/telemetry.py +++ b/llama_stack/apis/telemetry/telemetry.py @@ -169,39 +169,57 @@ class QueryCondition(BaseModel): value: Any +class QueryTracesResponse(BaseModel): + data: List[Trace] + + +class QuerySpansResponse(BaseModel): + data: List[Span] + + +class QuerySpanTreeResponse(BaseModel): + data: Dict[str, SpanWithStatus] + + @runtime_checkable class Telemetry(Protocol): - @webmethod(route="/telemetry/log-event") + @webmethod(route="/telemetry/events", method="POST") async def log_event( self, event: Event, ttl_seconds: int = DEFAULT_TTL_DAYS * 86400 ) -> None: ... - @webmethod(route="/telemetry/query-traces", method="POST") + @webmethod(route="/telemetry/traces", method="GET") async def query_traces( self, attribute_filters: Optional[List[QueryCondition]] = None, limit: Optional[int] = 100, offset: Optional[int] = 0, order_by: Optional[List[str]] = None, - ) -> List[Trace]: ... + ) -> QueryTracesResponse: ... - @webmethod(route="/telemetry/get-span-tree", method="POST") + @webmethod(route="/telemetry/traces/{trace_id}", method="GET") + async def get_trace(self, trace_id: str) -> Trace: ... + + @webmethod(route="/telemetry/traces/{trace_id}/spans/{span_id}", method="GET") + async def get_span(self, trace_id: str, span_id: str) -> Span: ... + + @webmethod(route="/telemetry/spans/{span_id}/tree", method="GET") async def get_span_tree( self, span_id: str, attributes_to_return: Optional[List[str]] = None, max_depth: Optional[int] = None, - ) -> Dict[str, SpanWithStatus]: ... + ) -> QuerySpanTreeResponse: ... - @webmethod(route="/telemetry/query-spans", method="POST") + @webmethod(route="/telemetry/spans", method="GET") async def query_spans( self, attribute_filters: List[QueryCondition], attributes_to_return: List[str], max_depth: Optional[int] = None, - ) -> List[Span]: ... + ) -> QuerySpansResponse: ... - @webmethod(route="/telemetry/save-spans-to-dataset", method="POST") + @webmethod(route="/telemetry/spans/export", method="POST") async def save_spans_to_dataset( self, attribute_filters: List[QueryCondition], diff --git a/llama_stack/apis/tools/__init__.py b/llama_stack/apis/tools/__init__.py index f747fcdc2..8cd798ebf 100644 --- a/llama_stack/apis/tools/__init__.py +++ b/llama_stack/apis/tools/__init__.py @@ -5,3 +5,4 @@ # the root directory of this source tree. from .tools import * # noqa: F401 F403 +from .rag_tool import * # noqa: F401 F403 diff --git a/llama_stack/apis/tools/rag_tool.py b/llama_stack/apis/tools/rag_tool.py new file mode 100644 index 000000000..950367304 --- /dev/null +++ b/llama_stack/apis/tools/rag_tool.py @@ -0,0 +1,95 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +from enum import Enum +from typing import Any, Dict, List, Literal, Optional, Union + +from llama_models.schema_utils import json_schema_type, register_schema, webmethod +from pydantic import BaseModel, Field +from typing_extensions import Annotated, Protocol, runtime_checkable + +from llama_stack.apis.common.content_types import InterleavedContent, URL +from llama_stack.providers.utils.telemetry.trace_protocol import trace_protocol + + +@json_schema_type +class RAGDocument(BaseModel): + document_id: str + content: InterleavedContent | URL + mime_type: str | None = None + metadata: Dict[str, Any] = Field(default_factory=dict) + + +@json_schema_type +class RAGQueryResult(BaseModel): + content: Optional[InterleavedContent] = None + + +@json_schema_type +class RAGQueryGenerator(Enum): + default = "default" + llm = "llm" + custom = "custom" + + +@json_schema_type +class DefaultRAGQueryGeneratorConfig(BaseModel): + type: Literal["default"] = "default" + separator: str = " " + + +@json_schema_type +class LLMRAGQueryGeneratorConfig(BaseModel): + type: Literal["llm"] = "llm" + model: str + template: str + + +RAGQueryGeneratorConfig = register_schema( + Annotated[ + Union[ + DefaultRAGQueryGeneratorConfig, + LLMRAGQueryGeneratorConfig, + ], + Field(discriminator="type"), + ], + name="RAGQueryGeneratorConfig", +) + + +@json_schema_type +class RAGQueryConfig(BaseModel): + # This config defines how a query is generated using the messages + # for memory bank retrieval. + query_generator_config: RAGQueryGeneratorConfig = Field( + default=DefaultRAGQueryGeneratorConfig() + ) + max_tokens_in_context: int = 4096 + max_chunks: int = 5 + + +@runtime_checkable +@trace_protocol +class RAGToolRuntime(Protocol): + @webmethod(route="/tool-runtime/rag-tool/insert", method="POST") + async def insert( + self, + documents: List[RAGDocument], + vector_db_id: str, + chunk_size_in_tokens: int = 512, + ) -> None: + """Index documents so they can be used by the RAG system""" + ... + + @webmethod(route="/tool-runtime/rag-tool/query", method="POST") + async def query( + self, + content: InterleavedContent, + vector_db_ids: List[str], + query_config: Optional[RAGQueryConfig] = None, + ) -> RAGQueryResult: + """Query the RAG system for context; typically invoked by the agent""" + ... diff --git a/llama_stack/apis/tools/tools.py b/llama_stack/apis/tools/tools.py index d2bdf9873..1af019bd4 100644 --- a/llama_stack/apis/tools/tools.py +++ b/llama_stack/apis/tools/tools.py @@ -15,6 +15,8 @@ from llama_stack.apis.common.content_types import InterleavedContent, URL from llama_stack.apis.resource import Resource, ResourceType from llama_stack.providers.utils.telemetry.trace_protocol import trace_protocol +from .rag_tool import RAGToolRuntime + @json_schema_type class ToolParameter(BaseModel): @@ -74,13 +76,21 @@ class ToolInvocationResult(BaseModel): class ToolStore(Protocol): def get_tool(self, tool_name: str) -> Tool: ... - def get_tool_group(self, tool_group_id: str) -> ToolGroup: ... + def get_tool_group(self, toolgroup_id: str) -> ToolGroup: ... + + +class ListToolGroupsResponse(BaseModel): + data: List[ToolGroup] + + +class ListToolsResponse(BaseModel): + data: List[Tool] @runtime_checkable @trace_protocol class ToolGroups(Protocol): - @webmethod(route="/toolgroups/register", method="POST") + @webmethod(route="/toolgroups", method="POST") async def register_tool_group( self, toolgroup_id: str, @@ -91,36 +101,48 @@ class ToolGroups(Protocol): """Register a tool group""" ... - @webmethod(route="/toolgroups/get", method="GET") + @webmethod(route="/toolgroups/{toolgroup_id}", method="GET") async def get_tool_group( self, toolgroup_id: str, ) -> ToolGroup: ... - @webmethod(route="/toolgroups/list", method="GET") - async def list_tool_groups(self) -> List[ToolGroup]: + @webmethod(route="/toolgroups", method="GET") + async def list_tool_groups(self) -> ListToolGroupsResponse: """List tool groups with optional provider""" ... - @webmethod(route="/tools/list", method="GET") - async def list_tools(self, tool_group_id: Optional[str] = None) -> List[Tool]: + @webmethod(route="/tools", method="GET") + async def list_tools(self, toolgroup_id: Optional[str] = None) -> ListToolsResponse: """List tools with optional tool group""" ... - @webmethod(route="/tools/get", method="GET") - async def get_tool(self, tool_name: str) -> Tool: ... + @webmethod(route="/tools/{tool_name}", method="GET") + async def get_tool( + self, + tool_name: str, + ) -> Tool: ... - @webmethod(route="/toolgroups/unregister", method="POST") - async def unregister_tool_group(self, tool_group_id: str) -> None: + @webmethod(route="/toolgroups/{toolgroup_id}", method="DELETE") + async def unregister_toolgroup( + self, + toolgroup_id: str, + ) -> None: """Unregister a tool group""" ... +class SpecialToolGroup(Enum): + rag_tool = "rag_tool" + + @runtime_checkable @trace_protocol class ToolRuntime(Protocol): tool_store: ToolStore + rag_tool: RAGToolRuntime + # TODO: This needs to be renamed once OPEN API generator name conflict issue is fixed. @webmethod(route="/tool-runtime/list-tools", method="GET") async def list_runtime_tools( @@ -129,7 +151,7 @@ class ToolRuntime(Protocol): @webmethod(route="/tool-runtime/invoke", method="POST") async def invoke_tool( - self, tool_name: str, args: Dict[str, Any] + self, tool_name: str, kwargs: Dict[str, Any] ) -> ToolInvocationResult: """Run a tool with the given arguments""" ... diff --git a/llama_stack/apis/memory_banks/__init__.py b/llama_stack/apis/vector_dbs/__init__.py similarity index 81% rename from llama_stack/apis/memory_banks/__init__.py rename to llama_stack/apis/vector_dbs/__init__.py index 7511677ab..158241a6d 100644 --- a/llama_stack/apis/memory_banks/__init__.py +++ b/llama_stack/apis/vector_dbs/__init__.py @@ -4,4 +4,4 @@ # This source code is licensed under the terms described in the LICENSE file in # the root directory of this source tree. -from .memory_banks import * # noqa: F401 F403 +from .vector_dbs import * # noqa: F401 F403 diff --git a/llama_stack/apis/vector_dbs/vector_dbs.py b/llama_stack/apis/vector_dbs/vector_dbs.py new file mode 100644 index 000000000..4b782e2d5 --- /dev/null +++ b/llama_stack/apis/vector_dbs/vector_dbs.py @@ -0,0 +1,66 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +from typing import List, Literal, Optional, Protocol, runtime_checkable + +from llama_models.schema_utils import json_schema_type, webmethod +from pydantic import BaseModel + +from llama_stack.apis.resource import Resource, ResourceType +from llama_stack.providers.utils.telemetry.trace_protocol import trace_protocol + + +@json_schema_type +class VectorDB(Resource): + type: Literal[ResourceType.vector_db.value] = ResourceType.vector_db.value + + embedding_model: str + embedding_dimension: int + + @property + def vector_db_id(self) -> str: + return self.identifier + + @property + def provider_vector_db_id(self) -> str: + return self.provider_resource_id + + +class VectorDBInput(BaseModel): + vector_db_id: str + embedding_model: str + embedding_dimension: int + provider_vector_db_id: Optional[str] = None + + +class ListVectorDBsResponse(BaseModel): + data: List[VectorDB] + + +@runtime_checkable +@trace_protocol +class VectorDBs(Protocol): + @webmethod(route="/vector-dbs", method="GET") + async def list_vector_dbs(self) -> ListVectorDBsResponse: ... + + @webmethod(route="/vector-dbs/{vector_db_id}", method="GET") + async def get_vector_db( + self, + vector_db_id: str, + ) -> Optional[VectorDB]: ... + + @webmethod(route="/vector-dbs", method="POST") + async def register_vector_db( + self, + vector_db_id: str, + embedding_model: str, + embedding_dimension: Optional[int] = 384, + provider_id: Optional[str] = None, + provider_vector_db_id: Optional[str] = None, + ) -> VectorDB: ... + + @webmethod(route="/vector-dbs/{vector_db_id}", method="DELETE") + async def unregister_vector_db(self, vector_db_id: str) -> None: ... diff --git a/llama_stack/apis/vector_io/__init__.py b/llama_stack/apis/vector_io/__init__.py new file mode 100644 index 000000000..3fe4fa4b6 --- /dev/null +++ b/llama_stack/apis/vector_io/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +from .vector_io import * # noqa: F401 F403 diff --git a/llama_stack/apis/memory/memory.py b/llama_stack/apis/vector_io/vector_io.py similarity index 58% rename from llama_stack/apis/memory/memory.py rename to llama_stack/apis/vector_io/vector_io.py index 8096a107a..8feeaa6d4 100644 --- a/llama_stack/apis/memory/memory.py +++ b/llama_stack/apis/vector_io/vector_io.py @@ -13,55 +13,45 @@ from typing import Any, Dict, List, Optional, Protocol, runtime_checkable from llama_models.schema_utils import json_schema_type, webmethod from pydantic import BaseModel, Field -from llama_stack.apis.common.content_types import URL from llama_stack.apis.inference import InterleavedContent -from llama_stack.apis.memory_banks import MemoryBank +from llama_stack.apis.vector_dbs import VectorDB from llama_stack.providers.utils.telemetry.trace_protocol import trace_protocol -@json_schema_type -class MemoryBankDocument(BaseModel): - document_id: str - content: InterleavedContent | URL - mime_type: str | None = None - metadata: Dict[str, Any] = Field(default_factory=dict) - - class Chunk(BaseModel): content: InterleavedContent - token_count: int - document_id: str + metadata: Dict[str, Any] = Field(default_factory=dict) @json_schema_type -class QueryDocumentsResponse(BaseModel): +class QueryChunksResponse(BaseModel): chunks: List[Chunk] scores: List[float] -class MemoryBankStore(Protocol): - def get_memory_bank(self, bank_id: str) -> Optional[MemoryBank]: ... +class VectorDBStore(Protocol): + def get_vector_db(self, vector_db_id: str) -> Optional[VectorDB]: ... @runtime_checkable @trace_protocol -class Memory(Protocol): - memory_bank_store: MemoryBankStore +class VectorIO(Protocol): + vector_db_store: VectorDBStore - # this will just block now until documents are inserted, but it should + # this will just block now until chunks are inserted, but it should # probably return a Job instance which can be polled for completion - @webmethod(route="/memory/insert") - async def insert_documents( + @webmethod(route="/vector-io/insert", method="POST") + async def insert_chunks( self, - bank_id: str, - documents: List[MemoryBankDocument], + vector_db_id: str, + chunks: List[Chunk], ttl_seconds: Optional[int] = None, ) -> None: ... - @webmethod(route="/memory/query") - async def query_documents( + @webmethod(route="/vector-io/query", method="POST") + async def query_chunks( self, - bank_id: str, + vector_db_id: str, query: InterleavedContent, params: Optional[Dict[str, Any]] = None, - ) -> QueryDocumentsResponse: ... + ) -> QueryChunksResponse: ... diff --git a/llama_stack/apis/version.py b/llama_stack/apis/version.py index f178712ba..53ad6a854 100644 --- a/llama_stack/apis/version.py +++ b/llama_stack/apis/version.py @@ -4,4 +4,4 @@ # This source code is licensed under the terms described in the LICENSE file in # the root directory of this source tree. -LLAMA_STACK_API_VERSION = "alpha" +LLAMA_STACK_API_VERSION = "v1" diff --git a/llama_stack/cli/model/describe.py b/llama_stack/cli/model/describe.py index 70e72f7be..fc0190ca8 100644 --- a/llama_stack/cli/model/describe.py +++ b/llama_stack/cli/model/describe.py @@ -13,7 +13,6 @@ from termcolor import colored from llama_stack.cli.subcommand import Subcommand from llama_stack.cli.table import print_table -from llama_stack.distribution.utils.serialize import EnumEncoder class ModelDescribe(Subcommand): @@ -72,7 +71,7 @@ class ModelDescribe(Subcommand): rows.append( ( "Recommended sampling params", - json.dumps(sampling_params, cls=EnumEncoder, indent=4), + json.dumps(sampling_params, indent=4), ) ) diff --git a/llama_stack/cli/stack/_build.py b/llama_stack/cli/stack/_build.py new file mode 100644 index 000000000..16ca670f7 --- /dev/null +++ b/llama_stack/cli/stack/_build.py @@ -0,0 +1,307 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +import argparse +import importlib.resources +import json +import os +import shutil +import textwrap +from functools import lru_cache +from pathlib import Path +from typing import Dict, Optional + +import yaml +from prompt_toolkit import prompt +from prompt_toolkit.completion import WordCompleter +from prompt_toolkit.validation import Validator +from termcolor import cprint + +from llama_stack.cli.table import print_table + +from llama_stack.distribution.build import build_image, ImageType +from llama_stack.distribution.datatypes import ( + BuildConfig, + DistributionSpec, + Provider, + StackRunConfig, +) +from llama_stack.distribution.distribution import get_provider_registry +from llama_stack.distribution.resolver import InvalidProviderError +from llama_stack.distribution.utils.config_dirs import DISTRIBS_BASE_DIR +from llama_stack.distribution.utils.dynamic import instantiate_class_type +from llama_stack.providers.datatypes import Api + + +TEMPLATES_PATH = Path(__file__).parent.parent.parent / "templates" + + +@lru_cache() +def available_templates_specs() -> Dict[str, BuildConfig]: + import yaml + + template_specs = {} + for p in TEMPLATES_PATH.rglob("*build.yaml"): + template_name = p.parent.name + with open(p, "r") as f: + build_config = BuildConfig(**yaml.safe_load(f)) + template_specs[template_name] = build_config + return template_specs + + +def run_stack_build_command( + parser: argparse.ArgumentParser, args: argparse.Namespace +) -> None: + if args.list_templates: + return _run_template_list_cmd() + + current_conda_env = os.environ.get("CONDA_DEFAULT_ENV") + image_name = args.image_name or current_conda_env + + if args.template: + available_templates = available_templates_specs() + if args.template not in available_templates: + cprint( + f"Could not find template {args.template}. Please run `llama stack build --list-templates` to check out the available templates", + color="red", + ) + return + build_config = available_templates[args.template] + if args.image_type: + build_config.image_type = args.image_type + else: + cprint( + f"Please specify a image-type (docker | conda | venv) for {args.template}", + color="red", + ) + return + _run_stack_build_command_from_build_config( + build_config, + image_name=image_name, + template_name=args.template, + ) + return + + if not args.config and not args.template: + name = prompt( + "> Enter a name for your Llama Stack (e.g. my-local-stack): ", + validator=Validator.from_callable( + lambda x: len(x) > 0, + error_message="Name cannot be empty, please enter a name", + ), + ) + + image_type = prompt( + "> Enter the image type you want your Llama Stack to be built as (docker or conda or venv): ", + validator=Validator.from_callable( + lambda x: x in ["docker", "conda", "venv"], + error_message="Invalid image type, please enter conda or docker or venv", + ), + default="conda", + ) + + if image_type == "conda": + if not image_name: + cprint( + f"No current conda environment detected or specified, will create a new conda environment with the name `llamastack-{name}`", + color="yellow", + ) + image_name = f"llamastack-{name}" + else: + cprint( + f"Using conda environment {image_name}", + color="green", + ) + + cprint( + textwrap.dedent( + """ + Llama Stack is composed of several APIs working together. Let's select + the provider types (implementations) you want to use for these APIs. + """, + ), + color="green", + ) + + print("Tip: use to see options for the providers.\n") + + providers = dict() + for api, providers_for_api in get_provider_registry().items(): + available_providers = [ + x + for x in providers_for_api.keys() + if x not in ("remote", "remote::sample") + ] + api_provider = prompt( + "> Enter provider for API {}: ".format(api.value), + completer=WordCompleter(available_providers), + complete_while_typing=True, + validator=Validator.from_callable( + lambda x: x in available_providers, + error_message="Invalid provider, use to see options", + ), + ) + + providers[api.value] = api_provider + + description = prompt( + "\n > (Optional) Enter a short description for your Llama Stack: ", + default="", + ) + + distribution_spec = DistributionSpec( + providers=providers, + description=description, + ) + + build_config = BuildConfig( + image_type=image_type, distribution_spec=distribution_spec + ) + else: + with open(args.config, "r") as f: + try: + build_config = BuildConfig(**yaml.safe_load(f)) + except Exception as e: + cprint( + f"Could not parse config file {args.config}: {e}", + color="red", + ) + return + + _run_stack_build_command_from_build_config(build_config, image_name=image_name) + + +def _generate_run_config( + build_config: BuildConfig, build_dir: Path, image_name: str +) -> None: + """ + Generate a run.yaml template file for user to edit from a build.yaml file + """ + apis = list(build_config.distribution_spec.providers.keys()) + run_config = StackRunConfig( + container_image=( + image_name if build_config.image_type == ImageType.container.value else None + ), + image_name=image_name, + apis=apis, + providers={}, + ) + # build providers dict + provider_registry = get_provider_registry() + for api in apis: + run_config.providers[api] = [] + provider_types = build_config.distribution_spec.providers[api] + if isinstance(provider_types, str): + provider_types = [provider_types] + + for i, provider_type in enumerate(provider_types): + pid = provider_type.split("::")[-1] + + p = provider_registry[Api(api)][provider_type] + if p.deprecation_error: + raise InvalidProviderError(p.deprecation_error) + + config_type = instantiate_class_type( + provider_registry[Api(api)][provider_type].config_class + ) + if hasattr(config_type, "sample_run_config"): + config = config_type.sample_run_config( + __distro_dir__=f"distributions/{image_name}" + ) + else: + config = {} + + p_spec = Provider( + provider_id=f"{pid}-{i}" if len(provider_types) > 1 else pid, + provider_type=provider_type, + config=config, + ) + run_config.providers[api].append(p_spec) + + run_config_file = build_dir / f"{image_name}-run.yaml" + + with open(run_config_file, "w") as f: + to_write = json.loads(run_config.model_dump_json()) + f.write(yaml.dump(to_write, sort_keys=False)) + + cprint( + f"You can now edit {run_config_file} and run `llama stack run {image_name}`", + color="green", + ) + + +def _run_stack_build_command_from_build_config( + build_config: BuildConfig, + image_name: Optional[str] = None, + template_name: Optional[str] = None, +) -> None: + if build_config.image_type == ImageType.container.value: + if template_name: + image_name = f"distribution-{template_name}" + else: + if not image_name: + raise ValueError( + "Please specify an image name when building a docker image without a template" + ) + elif build_config.image_type == ImageType.conda.value: + if not image_name: + raise ValueError("Please specify an image name when building a conda image") + + if template_name: + build_dir = DISTRIBS_BASE_DIR / template_name + build_file_path = build_dir / f"{template_name}-build.yaml" + else: + build_dir = DISTRIBS_BASE_DIR / image_name + build_file_path = build_dir / f"{image_name}-build.yaml" + + os.makedirs(build_dir, exist_ok=True) + with open(build_file_path, "w") as f: + to_write = json.loads(build_config.model_dump_json()) + f.write(yaml.dump(to_write, sort_keys=False)) + + return_code = build_image( + build_config, build_file_path, image_name, template_name=template_name + ) + if return_code != 0: + return + + if template_name: + # copy run.yaml from template to build_dir instead of generating it again + template_path = ( + importlib.resources.files("llama_stack") + / f"templates/{template_name}/run.yaml" + ) + with importlib.resources.as_file(template_path) as path: + run_config_file = build_dir / f"{template_name}-run.yaml" + shutil.copy(path, run_config_file) + # Find all ${env.VARIABLE} patterns + cprint("Build Successful!", color="green") + else: + _generate_run_config(build_config, build_dir, image_name) + + +def _run_template_list_cmd() -> None: + # eventually, this should query a registry at llama.meta.com/llamastack/distributions + headers = [ + "Template Name", + # "Providers", + "Description", + ] + + rows = [] + for template_name, spec in available_templates_specs().items(): + rows.append( + [ + template_name, + # json.dumps(spec.distribution_spec.providers, indent=2), + spec.distribution_spec.description, + ] + ) + print_table( + rows, + headers, + separate_rows=True, + ) diff --git a/llama_stack/cli/stack/build.py b/llama_stack/cli/stack/build.py index 084374c8a..48c811839 100644 --- a/llama_stack/cli/stack/build.py +++ b/llama_stack/cli/stack/build.py @@ -4,43 +4,10 @@ # This source code is licensed under the terms described in the LICENSE file in # the root directory of this source tree. import argparse - -import importlib.resources - -import os -import shutil -from functools import lru_cache -from pathlib import Path -from typing import List, Optional +import textwrap from llama_stack.cli.subcommand import Subcommand -from llama_stack.distribution.datatypes import ( - BuildConfig, - DistributionSpec, - Provider, - StackRunConfig, -) - -from llama_stack.distribution.distribution import get_provider_registry -from llama_stack.distribution.resolver import InvalidProviderError -from llama_stack.distribution.utils.dynamic import instantiate_class_type -from llama_stack.providers.datatypes import Api - -TEMPLATES_PATH = Path(__file__).parent.parent.parent / "templates" - - -@lru_cache() -def available_templates_specs() -> List[BuildConfig]: - import yaml - - template_specs = [] - for p in TEMPLATES_PATH.rglob("*build.yaml"): - with open(p, "r") as f: - build_config = BuildConfig(**yaml.safe_load(f)) - template_specs.append(build_config) - return template_specs - class StackBuild(Subcommand): def __init__(self, subparsers: argparse._SubParsersAction): @@ -80,251 +47,26 @@ class StackBuild(Subcommand): self.parser.add_argument( "--image-type", type=str, - help="Image Type to use for the build. This can be either conda or docker. If not specified, will use the image type from the template config.", - choices=["conda", "docker", "venv"], + help="Image Type to use for the build. This can be either conda or container or venv. If not specified, will use the image type from the template config.", + choices=["conda", "container", "venv"], default="conda", ) + self.parser.add_argument( + "--image-name", + type=str, + help=textwrap.dedent( + """[for image-type=conda] Name of the conda environment to use for the build. If +not specified, currently active Conda environment will be used. If no Conda +environment is active, you must specify a name. + """ + ), + default=None, + ) + def _run_stack_build_command(self, args: argparse.Namespace) -> None: - import textwrap + # always keep implementation completely silo-ed away from CLI so CLI + # can be fast to load and reduces dependencies + from ._build import run_stack_build_command - import yaml - from prompt_toolkit import prompt - from prompt_toolkit.completion import WordCompleter - from prompt_toolkit.validation import Validator - from termcolor import cprint - - from llama_stack.distribution.distribution import get_provider_registry - - if args.list_templates: - self._run_template_list_cmd(args) - return - - if args.template: - available_templates = available_templates_specs() - for build_config in available_templates: - if build_config.name == args.template: - if args.image_type: - build_config.image_type = args.image_type - else: - self.parser.error( - f"Please specify a image-type (docker | conda | venv) for {args.template}" - ) - self._run_stack_build_command_from_build_config( - build_config, template_name=args.template - ) - return - - self.parser.error( - f"Could not find template {args.template}. Please run `llama stack build --list-templates` to check out the available templates" - ) - return - - if not args.config and not args.template: - name = prompt( - "> Enter a name for your Llama Stack (e.g. my-local-stack): ", - validator=Validator.from_callable( - lambda x: len(x) > 0, - error_message="Name cannot be empty, please enter a name", - ), - ) - - image_type = prompt( - "> Enter the image type you want your Llama Stack to be built as (docker or conda or venv): ", - validator=Validator.from_callable( - lambda x: x in ["docker", "conda", "venv"], - error_message="Invalid image type, please enter conda or docker or venv", - ), - default="conda", - ) - - cprint( - textwrap.dedent( - """ - Llama Stack is composed of several APIs working together. Let's select - the provider types (implementations) you want to use for these APIs. - """, - ), - color="green", - ) - - print("Tip: use to see options for the providers.\n") - - providers = dict() - for api, providers_for_api in get_provider_registry().items(): - available_providers = [ - x - for x in providers_for_api.keys() - if x not in ("remote", "remote::sample") - ] - api_provider = prompt( - "> Enter provider for API {}: ".format(api.value), - completer=WordCompleter(available_providers), - complete_while_typing=True, - validator=Validator.from_callable( - lambda x: x in available_providers, - error_message="Invalid provider, use to see options", - ), - ) - - providers[api.value] = api_provider - - description = prompt( - "\n > (Optional) Enter a short description for your Llama Stack: ", - default="", - ) - - distribution_spec = DistributionSpec( - providers=providers, - description=description, - ) - - build_config = BuildConfig( - name=name, image_type=image_type, distribution_spec=distribution_spec - ) - self._run_stack_build_command_from_build_config(build_config) - return - - with open(args.config, "r") as f: - try: - build_config = BuildConfig(**yaml.safe_load(f)) - except Exception as e: - self.parser.error(f"Could not parse config file {args.config}: {e}") - return - self._run_stack_build_command_from_build_config(build_config) - - def _generate_run_config(self, build_config: BuildConfig, build_dir: Path) -> None: - """ - Generate a run.yaml template file for user to edit from a build.yaml file - """ - import json - - import yaml - from termcolor import cprint - - from llama_stack.distribution.build import ImageType - - apis = list(build_config.distribution_spec.providers.keys()) - run_config = StackRunConfig( - docker_image=( - build_config.name - if build_config.image_type == ImageType.docker.value - else None - ), - image_name=build_config.name, - conda_env=( - build_config.name - if build_config.image_type == ImageType.conda.value - else None - ), - apis=apis, - providers={}, - ) - # build providers dict - provider_registry = get_provider_registry() - for api in apis: - run_config.providers[api] = [] - provider_types = build_config.distribution_spec.providers[api] - if isinstance(provider_types, str): - provider_types = [provider_types] - - for i, provider_type in enumerate(provider_types): - pid = provider_type.split("::")[-1] - - p = provider_registry[Api(api)][provider_type] - if p.deprecation_error: - raise InvalidProviderError(p.deprecation_error) - - config_type = instantiate_class_type( - provider_registry[Api(api)][provider_type].config_class - ) - if hasattr(config_type, "sample_run_config"): - config = config_type.sample_run_config( - __distro_dir__=f"distributions/{build_config.name}" - ) - else: - config = {} - - p_spec = Provider( - provider_id=f"{pid}-{i}" if len(provider_types) > 1 else pid, - provider_type=provider_type, - config=config, - ) - run_config.providers[api].append(p_spec) - - os.makedirs(build_dir, exist_ok=True) - run_config_file = build_dir / f"{build_config.name}-run.yaml" - - with open(run_config_file, "w") as f: - to_write = json.loads(run_config.model_dump_json()) - f.write(yaml.dump(to_write, sort_keys=False)) - - cprint( - f"You can now edit {run_config_file} and run `llama stack run {run_config_file}`", - color="green", - ) - - def _run_stack_build_command_from_build_config( - self, build_config: BuildConfig, template_name: Optional[str] = None - ) -> None: - import json - import os - - import yaml - from termcolor import cprint - - from llama_stack.distribution.build import build_image - from llama_stack.distribution.utils.config_dirs import DISTRIBS_BASE_DIR - - # save build.yaml spec for building same distribution again - build_dir = DISTRIBS_BASE_DIR / f"llamastack-{build_config.name}" - os.makedirs(build_dir, exist_ok=True) - build_file_path = build_dir / f"{build_config.name}-build.yaml" - - with open(build_file_path, "w") as f: - to_write = json.loads(build_config.model_dump_json()) - f.write(yaml.dump(to_write, sort_keys=False)) - - return_code = build_image(build_config, build_file_path) - if return_code != 0: - return - - if template_name: - # copy run.yaml from template to build_dir instead of generating it again - template_path = ( - importlib.resources.files("llama_stack") - / f"templates/{template_name}/run.yaml" - ) - with importlib.resources.as_file(template_path) as path: - shutil.copy(path, run_config_file) - # Find all ${env.VARIABLE} patterns - cprint("Build Successful!", color="green") - else: - self._generate_run_config(build_config, build_dir) - - def _run_template_list_cmd(self, args: argparse.Namespace) -> None: - import json - - from llama_stack.cli.table import print_table - - # eventually, this should query a registry at llama.meta.com/llamastack/distributions - headers = [ - "Template Name", - "Providers", - "Description", - ] - - rows = [] - for spec in available_templates_specs(): - rows.append( - [ - spec.name, - json.dumps(spec.distribution_spec.providers, indent=2), - spec.distribution_spec.description, - ] - ) - print_table( - rows, - headers, - separate_rows=True, - ) + return run_stack_build_command(self.parser, args) diff --git a/llama_stack/cli/stack/configure.py b/llama_stack/cli/stack/configure.py index 11d3f705a..56f4feceb 100644 --- a/llama_stack/cli/stack/configure.py +++ b/llama_stack/cli/stack/configure.py @@ -27,7 +27,7 @@ class StackConfigure(Subcommand): self.parser.add_argument( "config", type=str, - help="Path to the build config file (e.g. ~/.llama/builds//-build.yaml). For docker, this could also be the name of the docker image. ", + help="Path to the build config file (e.g. ~/.llama/builds//-build.yaml). For container, this could also be the name of the container image. ", ) self.parser.add_argument( diff --git a/llama_stack/cli/stack/run.py b/llama_stack/cli/stack/run.py index 90b2ecf6d..e1e02d10c 100644 --- a/llama_stack/cli/stack/run.py +++ b/llama_stack/cli/stack/run.py @@ -34,8 +34,13 @@ class StackRun(Subcommand): self.parser.add_argument( "--port", type=int, - help="Port to run the server on. Defaults to 5000", - default=int(os.getenv("LLAMA_STACK_PORT", 5000)), + help="Port to run the server on. Defaults to 8321", + default=int(os.getenv("LLAMA_STACK_PORT", 8321)), + ) + self.parser.add_argument( + "--image-name", + type=str, + help="Name of the image to run. Defaults to the current conda environment", ) self.parser.add_argument( "--disable-ipv6", @@ -53,8 +58,11 @@ class StackRun(Subcommand): def _run_stack_run_cmd(self, args: argparse.Namespace) -> None: import importlib.resources + import json + import subprocess import yaml + from termcolor import cprint from llama_stack.distribution.build import ImageType from llama_stack.distribution.configure import parse_and_maybe_upgrade_config @@ -84,9 +92,9 @@ class StackRun(Subcommand): ) if not config_file.exists() and not has_yaml_suffix: - # check if it's a build config saved to docker dir + # check if it's a build config saved to container dir config_file = Path( - BUILDS_BASE_DIR / ImageType.docker.value / f"{args.config}-run.yaml" + BUILDS_BASE_DIR / ImageType.container.value / f"{args.config}-run.yaml" ) if not config_file.exists() and not has_yaml_suffix: @@ -99,28 +107,67 @@ class StackRun(Subcommand): if not config_file.exists(): self.parser.error( - f"File {str(config_file)} does not exist. Please run `llama stack build` to generate (and optionally edit) a run.yaml file" + f"File {str(config_file)} does not exist.\n\nPlease run `llama stack build` to generate (and optionally edit) a run.yaml file" ) return - print(f"Using config file: {config_file}") + print(f"Using run configuration: {config_file}") config_dict = yaml.safe_load(config_file.read_text()) config = parse_and_maybe_upgrade_config(config_dict) - if config.docker_image: + if config.container_image: script = ( importlib.resources.files("llama_stack") / "distribution/start_container.sh" ) - run_args = [script, config.docker_image] + run_args = [script, config.container_image] else: + current_conda_env = os.environ.get("CONDA_DEFAULT_ENV") + image_name = args.image_name or current_conda_env + if not image_name: + cprint( + "No current conda environment detected, please specify a conda environment name with --image-name", + color="red", + ) + return + + def get_conda_prefix(env_name): + # Get conda environments info + conda_env_info = json.loads( + subprocess.check_output( + ["conda", "info", "--envs", "--json"] + ).decode() + ) + envs = conda_env_info["envs"] + for envpath in envs: + if envpath.endswith(env_name): + return envpath + return None + + print(f"Using conda environment: {image_name}") + conda_prefix = get_conda_prefix(image_name) + if not conda_prefix: + cprint( + f"Conda environment {image_name} does not exist.", + color="red", + ) + return + + build_file = Path(conda_prefix) / "llamastack-build.yaml" + if not build_file.exists(): + cprint( + f"Build file {build_file} does not exist.\n\nPlease run `llama stack build` or specify the correct conda environment name with --image-name", + color="red", + ) + return + script = ( importlib.resources.files("llama_stack") / "distribution/start_conda_env.sh" ) run_args = [ script, - config.conda_env, + image_name, ] run_args.extend([str(config_file), str(args.port)]) @@ -129,13 +176,17 @@ class StackRun(Subcommand): for env_var in args.env: if "=" not in env_var: - self.parser.error( - f"Environment variable '{env_var}' must be in KEY=VALUE format" + cprint( + f"Environment variable '{env_var}' must be in KEY=VALUE format", + color="red", ) return key, value = env_var.split("=", 1) # split on first = only if not key: - self.parser.error(f"Environment variable '{env_var}' has empty key") + cprint( + f"Environment variable '{env_var}' has empty key", + color="red", + ) return run_args.extend(["--env", f"{key}={value}"]) diff --git a/llama_stack/distribution/build.py b/llama_stack/distribution/build.py index 5a7dfba11..950338730 100644 --- a/llama_stack/distribution/build.py +++ b/llama_stack/distribution/build.py @@ -6,10 +6,11 @@ import importlib.resources import logging +import sys from enum import Enum from pathlib import Path -from typing import Dict, List +from typing import Dict, List, Optional from pydantic import BaseModel from termcolor import cprint @@ -20,7 +21,7 @@ from llama_stack.distribution.distribution import get_provider_registry from llama_stack.distribution.utils.config_dirs import BUILDS_BASE_DIR -from llama_stack.distribution.utils.exec import run_with_pty +from llama_stack.distribution.utils.exec import run_command, run_with_pty from llama_stack.providers.datatypes import Api log = logging.getLogger(__name__) @@ -37,7 +38,7 @@ SERVER_DEPENDENCIES = [ class ImageType(Enum): - docker = "docker" + container = "container" conda = "conda" venv = "venv" @@ -76,8 +77,8 @@ def get_provider_dependencies( provider_spec = providers_for_api[provider_type] deps.extend(provider_spec.pip_packages) - if provider_spec.docker_image: - raise ValueError("A stack's dependencies cannot have a docker image") + if provider_spec.container_image: + raise ValueError("A stack's dependencies cannot have a container image") normal_deps = [] special_deps = [] @@ -102,41 +103,53 @@ def print_pip_install_help(providers: Dict[str, List[Provider]]): print() -def build_image(build_config: BuildConfig, build_file_path: Path): - docker_image = build_config.distribution_spec.docker_image or "python:3.10-slim" +def build_image( + build_config: BuildConfig, + build_file_path: Path, + image_name: str, + template_name: Optional[str] = None, +): + container_image = ( + build_config.distribution_spec.container_image or "python:3.10-slim" + ) normal_deps, special_deps = get_provider_dependencies( build_config.distribution_spec.providers ) normal_deps += SERVER_DEPENDENCIES - if build_config.image_type == ImageType.docker.value: - script = ( + if build_config.image_type == ImageType.container.value: + if not template_name: + raise ValueError("template_name is required for container builds") + + script = str( importlib.resources.files("llama_stack") / "distribution/build_container.sh" ) args = [ script, - build_config.name, - docker_image, + template_name, + container_image, str(build_file_path), - str(BUILDS_BASE_DIR / ImageType.docker.value), + str(BUILDS_BASE_DIR / ImageType.container.value), " ".join(normal_deps), ] elif build_config.image_type == ImageType.conda.value: - script = ( + script = str( importlib.resources.files("llama_stack") / "distribution/build_conda_env.sh" ) args = [ script, - build_config.name, + str(image_name), str(build_file_path), " ".join(normal_deps), ] elif build_config.image_type == ImageType.venv.value: - script = importlib.resources.files("llama_stack") / "distribution/build_venv.sh" + script = str( + importlib.resources.files("llama_stack") / "distribution/build_venv.sh" + ) args = [ script, - build_config.name, + str(image_name), str(build_file_path), " ".join(normal_deps), ] @@ -144,10 +157,15 @@ def build_image(build_config: BuildConfig, build_file_path: Path): if special_deps: args.append("#".join(special_deps)) - return_code = run_with_pty(args) + is_terminal = sys.stdin.isatty() + if is_terminal: + return_code = run_with_pty(args) + else: + return_code = run_command(args) + if return_code != 0: log.error( - f"Failed to build target {build_config.name} with return code {return_code}", + f"Failed to build target {image_name} with return code {return_code}", ) return return_code diff --git a/llama_stack/distribution/build_conda_env.sh b/llama_stack/distribution/build_conda_env.sh index fc1e48665..606fbf19d 100755 --- a/llama_stack/distribution/build_conda_env.sh +++ b/llama_stack/distribution/build_conda_env.sh @@ -18,8 +18,8 @@ if [ -n "$LLAMA_MODELS_DIR" ]; then fi if [ "$#" -lt 3 ]; then - echo "Usage: $0 []" >&2 - echo "Example: $0 mybuild ./my-stack-build.yaml 'numpy pandas scipy'" >&2 + echo "Usage: $0 []" >&2 + echo "Example: $0 my-conda-env ./my-stack-build.yaml 'numpy pandas scipy'" >&2 exit 1 fi @@ -27,8 +27,7 @@ special_pip_deps="$4" set -euo pipefail -build_name="$1" -env_name="llamastack-$build_name" +env_name="$1" build_file_path="$2" pip_dependencies="$3" @@ -105,7 +104,13 @@ ensure_conda_env_python310() { printf "Installing from LLAMA_STACK_DIR: $LLAMA_STACK_DIR\n" $CONDA_PREFIX/bin/pip install --no-cache-dir -e "$LLAMA_STACK_DIR" else - $CONDA_PREFIX/bin/pip install --no-cache-dir llama-stack + PYPI_VERSION="${PYPI_VERSION:-}" + if [ -n "$PYPI_VERSION" ]; then + SPEC_VERSION="llama-stack==${PYPI_VERSION} llama-models==${PYPI_VERSION} llama-stack-client==${PYPI_VERSION}" + else + SPEC_VERSION="llama-stack" + fi + $CONDA_PREFIX/bin/pip install --no-cache-dir $SPEC_VERSION fi if [ -n "$LLAMA_MODELS_DIR" ]; then @@ -131,8 +136,8 @@ ensure_conda_env_python310() { fi fi - mv $build_file_path $CONDA_PREFIX/ - echo "Build spec configuration saved at $CONDA_PREFIX/$build_name-build.yaml" + mv $build_file_path $CONDA_PREFIX/llamastack-build.yaml + echo "Build spec configuration saved at $CONDA_PREFIX/llamastack-build.yaml" } ensure_conda_env_python310 "$env_name" "$pip_dependencies" "$special_pip_deps" diff --git a/llama_stack/distribution/build_container.sh b/llama_stack/distribution/build_container.sh index 286ade992..91c1dd1a6 100755 --- a/llama_stack/distribution/build_container.sh +++ b/llama_stack/distribution/build_container.sh @@ -9,11 +9,13 @@ LLAMA_MODELS_DIR=${LLAMA_MODELS_DIR:-} LLAMA_STACK_DIR=${LLAMA_STACK_DIR:-} TEST_PYPI_VERSION=${TEST_PYPI_VERSION:-} +PYPI_VERSION=${PYPI_VERSION:-} BUILD_PLATFORM=${BUILD_PLATFORM:-} -if [ "$#" -lt 4 ]; then - echo "Usage: $0 []" >&2 - echo "Example: $0 my-fastapi-app python:3.9-slim 'fastapi uvicorn' " >&2 +if [ "$#" -lt 5 ]; then + # This only works for templates + echo "Usage: $0 []" >&2 + echo "Example: $0 fireworks python:3.9-slim 'fastapi uvicorn' /path/to/build/dir" >&2 exit 1 fi @@ -21,9 +23,8 @@ special_pip_deps="$6" set -euo pipefail -build_name="$1" -image_name="distribution-$build_name" -docker_base=$2 +template_name="$1" +container_base=$2 build_file_path=$3 host_build_dir=$4 pip_dependencies=$5 @@ -35,14 +36,14 @@ NC='\033[0m' # No Color SCRIPT_DIR=$(dirname "$(readlink -f "$0")") REPO_DIR=$(dirname $(dirname "$SCRIPT_DIR")) -DOCKER_BINARY=${DOCKER_BINARY:-docker} -DOCKER_OPTS=${DOCKER_OPTS:-} +CONTAINER_BINARY=${CONTAINER_BINARY:-docker} +CONTAINER_OPTS=${CONTAINER_OPTS:-} TEMP_DIR=$(mktemp -d) -add_to_docker() { +add_to_container() { local input - output_file="$TEMP_DIR/Dockerfile" + output_file="$TEMP_DIR/Containerfile" if [ -t 0 ]; then printf '%s\n' "$1" >>"$output_file" else @@ -52,9 +53,9 @@ add_to_docker() { } # Update and install UBI9 components if UBI9 base image is used -if [[ $docker_base == *"registry.access.redhat.com/ubi9"* ]]; then - add_to_docker </dev/null && selinuxenabled; then # Disable SELinux labels -- we don't want to relabel the llama-stack source dir - DOCKER_OPTS="$DOCKER_OPTS --security-opt label=disable" + CONTAINER_OPTS="$CONTAINER_OPTS --security-opt label=disable" fi # Set version tag based on PyPI version @@ -167,7 +184,8 @@ else fi # Add version tag to image name -image_tag="$image_name:$version_tag" +build_name="distribution-$template_name" +image_tag="$build_name:$version_tag" # Detect platform architecture ARCH=$(uname -m) @@ -183,7 +201,7 @@ else fi set -x -$DOCKER_BINARY build $DOCKER_OPTS $PLATFORM -t $image_tag -f "$TEMP_DIR/Dockerfile" "$REPO_DIR" $mounts +$CONTAINER_BINARY build $CONTAINER_OPTS $PLATFORM -t $image_tag -f "$TEMP_DIR/Containerfile" "$REPO_DIR" $mounts # clean up tmp/configs set +x diff --git a/llama_stack/distribution/configure_container.sh b/llama_stack/distribution/configure_container.sh index 5f64531eb..b01251e46 100755 --- a/llama_stack/distribution/configure_container.sh +++ b/llama_stack/distribution/configure_container.sh @@ -6,8 +6,8 @@ # This source code is licensed under the terms described in the LICENSE file in # the root directory of this source tree. -DOCKER_BINARY=${DOCKER_BINARY:-docker} -DOCKER_OPTS=${DOCKER_OPTS:-} +CONTAINER_BINARY=${CONTAINER_BINARY:-docker} +CONTAINER_OPTS=${CONTAINER_OPTS:-} LLAMA_STACK_DIR=${LLAMA_STACK_DIR:-} set -euo pipefail @@ -24,13 +24,13 @@ if [ $# -lt 2 ]; then exit 1 fi -docker_image="$1" +container_image="$1" host_build_dir="$2" container_build_dir="/app/builds" if command -v selinuxenabled &> /dev/null && selinuxenabled; then # Disable SELinux labels - DOCKER_OPTS="$DOCKER_OPTS --security-opt label=disable" + CONTAINER_OPTS="$CONTAINER_OPTS --security-opt label=disable" fi mounts="" @@ -39,9 +39,9 @@ if [ -n "$LLAMA_STACK_DIR" ]; then fi set -x -$DOCKER_BINARY run $DOCKER_OPTS -it \ +$CONTAINER_BINARY run $CONTAINER_OPTS -it \ --entrypoint "/usr/local/bin/llama" \ -v $host_build_dir:$container_build_dir \ $mounts \ - $docker_image \ + $container_image \ stack configure ./llamastack-build.yaml --output-dir $container_build_dir diff --git a/llama_stack/distribution/datatypes.py b/llama_stack/distribution/datatypes.py index d0ccd6cd1..99ffeb346 100644 --- a/llama_stack/distribution/datatypes.py +++ b/llama_stack/distribution/datatypes.py @@ -13,14 +13,14 @@ from llama_stack.apis.datasets import Dataset, DatasetInput from llama_stack.apis.eval import Eval from llama_stack.apis.eval_tasks import EvalTask, EvalTaskInput from llama_stack.apis.inference import Inference -from llama_stack.apis.memory import Memory -from llama_stack.apis.memory_banks import MemoryBank, MemoryBankInput from llama_stack.apis.models import Model, ModelInput from llama_stack.apis.safety import Safety from llama_stack.apis.scoring import Scoring from llama_stack.apis.scoring_functions import ScoringFn, ScoringFnInput from llama_stack.apis.shields import Shield, ShieldInput from llama_stack.apis.tools import Tool, ToolGroup, ToolGroupInput, ToolRuntime +from llama_stack.apis.vector_dbs import VectorDB, VectorDBInput +from llama_stack.apis.vector_io import VectorIO from llama_stack.providers.datatypes import Api, ProviderSpec from llama_stack.providers.utils.kvstore.config import KVStoreConfig @@ -34,7 +34,7 @@ RoutingKey = Union[str, List[str]] RoutableObject = Union[ Model, Shield, - MemoryBank, + VectorDB, Dataset, ScoringFn, EvalTask, @@ -47,7 +47,7 @@ RoutableObjectWithProvider = Annotated[ Union[ Model, Shield, - MemoryBank, + VectorDB, Dataset, ScoringFn, EvalTask, @@ -60,7 +60,7 @@ RoutableObjectWithProvider = Annotated[ RoutedProtocol = Union[ Inference, Safety, - Memory, + VectorIO, DatasetIO, Scoring, Eval, @@ -73,7 +73,7 @@ class AutoRoutedProviderSpec(ProviderSpec): provider_type: str = "router" config_class: str = "" - docker_image: Optional[str] = None + container_image: Optional[str] = None routing_table_api: Api module: str provider_data_validator: Optional[str] = Field( @@ -89,7 +89,7 @@ class AutoRoutedProviderSpec(ProviderSpec): class RoutingTableProviderSpec(ProviderSpec): provider_type: str = "routing_table" config_class: str = "" - docker_image: Optional[str] = None + container_image: Optional[str] = None router_api: Api module: str @@ -101,7 +101,7 @@ class DistributionSpec(BaseModel): default="", description="Description of the distribution", ) - docker_image: Optional[str] = None + container_image: Optional[str] = None providers: Dict[str, Union[str, List[str]]] = Field( default_factory=dict, description=""" @@ -127,13 +127,9 @@ Reference to the distribution this package refers to. For unregistered (adhoc) p this could be just a hash """, ) - docker_image: Optional[str] = Field( + container_image: Optional[str] = Field( default=None, - description="Reference to the docker image if this package refers to a container", - ) - conda_env: Optional[str] = Field( - default=None, - description="Reference to the conda environment if this package refers to a conda environment", + description="Reference to the container image if this package refers to a container", ) apis: List[str] = Field( default_factory=list, @@ -157,7 +153,7 @@ a default SQLite store will be used.""", # registry of "resources" in the distribution models: List[ModelInput] = Field(default_factory=list) shields: List[ShieldInput] = Field(default_factory=list) - memory_banks: List[MemoryBankInput] = Field(default_factory=list) + vector_dbs: List[VectorDBInput] = Field(default_factory=list) datasets: List[DatasetInput] = Field(default_factory=list) scoring_fns: List[ScoringFnInput] = Field(default_factory=list) eval_tasks: List[EvalTaskInput] = Field(default_factory=list) @@ -166,11 +162,11 @@ a default SQLite store will be used.""", class BuildConfig(BaseModel): version: str = LLAMA_STACK_BUILD_CONFIG_VERSION - name: str + distribution_spec: DistributionSpec = Field( description="The distribution spec to build including API providers. " ) image_type: str = Field( default="conda", - description="Type of package to build (conda | docker | venv)", + description="Type of package to build (conda | container | venv)", ) diff --git a/llama_stack/distribution/distribution.py b/llama_stack/distribution/distribution.py index 4183d92cd..b02d0fb6c 100644 --- a/llama_stack/distribution/distribution.py +++ b/llama_stack/distribution/distribution.py @@ -32,8 +32,8 @@ def builtin_automatically_routed_apis() -> List[AutoRoutedApiInfo]: router_api=Api.safety, ), AutoRoutedApiInfo( - routing_table_api=Api.memory_banks, - router_api=Api.memory, + routing_table_api=Api.vector_dbs, + router_api=Api.vector_io, ), AutoRoutedApiInfo( routing_table_api=Api.datasets, diff --git a/llama_stack/distribution/inspect.py b/llama_stack/distribution/inspect.py index d275a5c2f..b7ee4a219 100644 --- a/llama_stack/distribution/inspect.py +++ b/llama_stack/distribution/inspect.py @@ -5,13 +5,14 @@ # the root directory of this source tree. from importlib.metadata import version -from typing import Dict, List from pydantic import BaseModel from llama_stack.apis.inspect import ( HealthInfo, Inspect, + ListProvidersResponse, + ListRoutesResponse, ProviderInfo, RouteInfo, VersionInfo, @@ -38,37 +39,43 @@ class DistributionInspectImpl(Inspect): async def initialize(self) -> None: pass - async def list_providers(self) -> Dict[str, List[ProviderInfo]]: + async def list_providers(self) -> ListProvidersResponse: run_config = self.config.run_config - ret = {} + ret = [] for api, providers in run_config.providers.items(): - ret[api] = [ - ProviderInfo( - provider_id=p.provider_id, - provider_type=p.provider_type, - ) - for p in providers - ] + ret.extend( + [ + ProviderInfo( + api=api, + provider_id=p.provider_id, + provider_type=p.provider_type, + ) + for p in providers + ] + ) - return ret + return ListProvidersResponse(data=ret) - async def list_routes(self) -> Dict[str, List[RouteInfo]]: + async def list_routes(self) -> ListRoutesResponse: run_config = self.config.run_config - ret = {} + ret = [] all_endpoints = get_all_api_endpoints() for api, endpoints in all_endpoints.items(): providers = run_config.providers.get(api.value, []) - ret[api.value] = [ - RouteInfo( - route=e.route, - method=e.method, - provider_types=[p.provider_type for p in providers], - ) - for e in endpoints - ] - return ret + ret.extend( + [ + RouteInfo( + route=e.route, + method=e.method, + provider_types=[p.provider_type for p in providers], + ) + for e in endpoints + ] + ) + + return ListRoutesResponse(data=ret) async def health(self) -> HealthInfo: return HealthInfo(status="OK") diff --git a/llama_stack/distribution/library_client.py b/llama_stack/distribution/library_client.py index 50af2cdea..192667f2c 100644 --- a/llama_stack/distribution/library_client.py +++ b/llama_stack/distribution/library_client.py @@ -9,12 +9,11 @@ import inspect import json import logging import os -import queue -import threading +import re from concurrent.futures import ThreadPoolExecutor from enum import Enum from pathlib import Path -from typing import Any, Generator, get_args, get_origin, Optional, TypeVar +from typing import Any, get_args, get_origin, Optional, TypeVar import httpx import yaml @@ -64,71 +63,6 @@ def in_notebook(): return True -def stream_across_asyncio_run_boundary( - async_gen_maker, - pool_executor: ThreadPoolExecutor, - path: Optional[str] = None, - provider_data: Optional[dict[str, Any]] = None, -) -> Generator[T, None, None]: - result_queue = queue.Queue() - stop_event = threading.Event() - - async def consumer(): - # make sure we make the generator in the event loop context - gen = await async_gen_maker() - await start_trace(path, {"__location__": "library_client"}) - if provider_data: - set_request_provider_data( - {"X-LlamaStack-Provider-Data": json.dumps(provider_data)} - ) - try: - async for item in await gen: - result_queue.put(item) - except Exception as e: - print(f"Error in generator {e}") - result_queue.put(e) - except asyncio.CancelledError: - return - finally: - result_queue.put(StopIteration) - stop_event.set() - await end_trace() - - def run_async(): - # Run our own loop to avoid double async generator cleanup which is done - # by asyncio.run() - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - try: - task = loop.create_task(consumer()) - loop.run_until_complete(task) - finally: - # Handle pending tasks like a generator's athrow() - pending = asyncio.all_tasks(loop) - if pending: - loop.run_until_complete( - asyncio.gather(*pending, return_exceptions=True) - ) - loop.close() - - future = pool_executor.submit(run_async) - - try: - # yield results as they come in - while not stop_event.is_set() or not result_queue.empty(): - try: - item = result_queue.get(timeout=0.1) - if item is StopIteration: - break - if isinstance(item, Exception): - raise item - yield item - except queue.Empty: - continue - finally: - future.result() - - def convert_pydantic_to_json_value(value: Any) -> Any: if isinstance(value, Enum): return value.value @@ -184,7 +118,7 @@ class LlamaStackAsLibraryClient(LlamaStackClient): ): super().__init__() self.async_client = AsyncLlamaStackAsLibraryClient( - config_path_or_template_name, custom_provider_registry + config_path_or_template_name, custom_provider_registry, provider_data ) self.pool_executor = ThreadPoolExecutor(max_workers=4) self.skip_logger_removal = skip_logger_removal @@ -210,39 +144,30 @@ class LlamaStackAsLibraryClient(LlamaStackClient): root_logger.removeHandler(handler) print(f"Removed handler {handler.__class__.__name__} from root logger") - def _get_path( - self, - cast_to: Any, - options: Any, - *, - stream=False, - stream_cls=None, - ): - return options.url - def request(self, *args, **kwargs): - path = self._get_path(*args, **kwargs) if kwargs.get("stream"): - return stream_across_asyncio_run_boundary( - lambda: self.async_client.request(*args, **kwargs), - self.pool_executor, - path=path, - provider_data=self.provider_data, - ) - else: + # NOTE: We are using AsyncLlamaStackClient under the hood + # A new event loop is needed to convert the AsyncStream + # from async client into SyncStream return type for streaming + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) - async def _traced_request(): - if self.provider_data: - set_request_provider_data( - {"X-LlamaStack-Provider-Data": json.dumps(self.provider_data)} - ) - await start_trace(path, {"__location__": "library_client"}) + def sync_generator(): try: - return await self.async_client.request(*args, **kwargs) + async_stream = loop.run_until_complete( + self.async_client.request(*args, **kwargs) + ) + while True: + chunk = loop.run_until_complete(async_stream.__anext__()) + yield chunk + except StopAsyncIteration: + pass finally: - await end_trace() + loop.close() - return asyncio.run(_traced_request()) + return sync_generator() + else: + return asyncio.run(self.async_client.request(*args, **kwargs)) class AsyncLlamaStackAsLibraryClient(AsyncLlamaStackClient): @@ -250,9 +175,9 @@ class AsyncLlamaStackAsLibraryClient(AsyncLlamaStackClient): self, config_path_or_template_name: str, custom_provider_registry: Optional[ProviderRegistry] = None, + provider_data: Optional[dict[str, Any]] = None, ): super().__init__() - # when using the library client, we should not log to console since many # of our logs are intended for server-side usage current_sinks = os.environ.get("TELEMETRY_SINKS", "sqlite").split(",") @@ -273,6 +198,7 @@ class AsyncLlamaStackAsLibraryClient(AsyncLlamaStackClient): self.config_path_or_template_name = config_path_or_template_name self.config = config self.custom_provider_registry = custom_provider_registry + self.provider_data = provider_data async def initialize(self): try: @@ -307,13 +233,23 @@ class AsyncLlamaStackAsLibraryClient(AsyncLlamaStackClient): endpoints = get_all_api_endpoints() endpoint_impls = {} + + def _convert_path_to_regex(path: str) -> str: + # Convert {param} to named capture groups + pattern = re.sub(r"{(\w+)}", r"(?P<\1>[^/]+)", path) + return f"^{pattern}$" + for api, api_endpoints in endpoints.items(): if api not in self.impls: continue for endpoint in api_endpoints: impl = self.impls[api] func = getattr(impl, endpoint.name) - endpoint_impls[endpoint.route] = func + if endpoint.method not in endpoint_impls: + endpoint_impls[endpoint.method] = {} + endpoint_impls[endpoint.method][ + _convert_path_to_regex(endpoint.route) + ] = func self.endpoint_impls = endpoint_impls return True @@ -329,17 +265,49 @@ class AsyncLlamaStackAsLibraryClient(AsyncLlamaStackClient): if not self.endpoint_impls: raise ValueError("Client not initialized") + if self.provider_data: + set_request_provider_data( + {"X-LlamaStack-Provider-Data": json.dumps(self.provider_data)} + ) + if stream: - return self._call_streaming( + response = await self._call_streaming( cast_to=cast_to, options=options, stream_cls=stream_cls, ) else: - return await self._call_non_streaming( + response = await self._call_non_streaming( cast_to=cast_to, options=options, ) + return response + + def _find_matching_endpoint(self, method: str, path: str) -> tuple[Any, dict]: + """Find the matching endpoint implementation for a given method and path. + + Args: + method: HTTP method (GET, POST, etc.) + path: URL path to match against + + Returns: + A tuple of (endpoint_function, path_params) + + Raises: + ValueError: If no matching endpoint is found + """ + impls = self.endpoint_impls.get(method) + if not impls: + raise ValueError(f"No endpoint found for {path}") + + for regex, func in impls.items(): + match = re.match(regex, path) + if match: + # Extract named groups from the regex match + path_params = match.groupdict() + return func, path_params + + raise ValueError(f"No endpoint found for {path}") async def _call_non_streaming( self, @@ -348,15 +316,17 @@ class AsyncLlamaStackAsLibraryClient(AsyncLlamaStackClient): options: Any, ): path = options.url - body = options.params or {} body |= options.json_data or {} - func = self.endpoint_impls.get(path) - if not func: - raise ValueError(f"No endpoint found for {path}") - body = self._convert_body(path, body) - result = await func(**body) + matched_func, path_params = self._find_matching_endpoint(options.method, path) + body |= path_params + body = self._convert_body(path, options.method, body) + await start_trace(options.url, {"__location__": "library_client"}) + try: + result = await matched_func(**body) + finally: + await end_trace() json_content = json.dumps(convert_pydantic_to_json_value(result)) mock_response = httpx.Response( @@ -393,17 +363,20 @@ class AsyncLlamaStackAsLibraryClient(AsyncLlamaStackClient): path = options.url body = options.params or {} body |= options.json_data or {} - func = self.endpoint_impls.get(path) - if not func: - raise ValueError(f"No endpoint found for {path}") + func, path_params = self._find_matching_endpoint(options.method, path) + body |= path_params - body = self._convert_body(path, body) + body = self._convert_body(path, options.method, body) async def gen(): - async for chunk in await func(**body): - data = json.dumps(convert_pydantic_to_json_value(chunk)) - sse_event = f"data: {data}\n\n" - yield sse_event.encode("utf-8") + await start_trace(options.url, {"__location__": "library_client"}) + try: + async for chunk in await func(**body): + data = json.dumps(convert_pydantic_to_json_value(chunk)) + sse_event = f"data: {data}\n\n" + yield sse_event.encode("utf-8") + finally: + await end_trace() mock_response = httpx.Response( status_code=httpx.codes.OK, @@ -435,11 +408,13 @@ class AsyncLlamaStackAsLibraryClient(AsyncLlamaStackClient): ) return await response.parse() - def _convert_body(self, path: str, body: Optional[dict] = None) -> dict: + def _convert_body( + self, path: str, method: str, body: Optional[dict] = None + ) -> dict: if not body: return {} - func = self.endpoint_impls[path] + func, _ = self._find_matching_endpoint(method, path) sig = inspect.signature(func) # Strip NOT_GIVENs to use the defaults in signature diff --git a/llama_stack/distribution/resolver.py b/llama_stack/distribution/resolver.py index d7e947a46..dd6d4be6f 100644 --- a/llama_stack/distribution/resolver.py +++ b/llama_stack/distribution/resolver.py @@ -15,8 +15,6 @@ from llama_stack.apis.eval import Eval from llama_stack.apis.eval_tasks import EvalTasks from llama_stack.apis.inference import Inference from llama_stack.apis.inspect import Inspect -from llama_stack.apis.memory import Memory -from llama_stack.apis.memory_banks import MemoryBanks from llama_stack.apis.models import Models from llama_stack.apis.post_training import PostTraining from llama_stack.apis.safety import Safety @@ -25,6 +23,8 @@ from llama_stack.apis.scoring_functions import ScoringFunctions from llama_stack.apis.shields import Shields from llama_stack.apis.telemetry import Telemetry from llama_stack.apis.tools import ToolGroups, ToolRuntime +from llama_stack.apis.vector_dbs import VectorDBs +from llama_stack.apis.vector_io import VectorIO from llama_stack.distribution.client import get_client_impl from llama_stack.distribution.datatypes import ( AutoRoutedProviderSpec, @@ -40,7 +40,6 @@ from llama_stack.providers.datatypes import ( DatasetsProtocolPrivate, EvalTasksProtocolPrivate, InlineProviderSpec, - MemoryBanksProtocolPrivate, ModelsProtocolPrivate, ProviderSpec, RemoteProviderConfig, @@ -48,6 +47,7 @@ from llama_stack.providers.datatypes import ( ScoringFunctionsProtocolPrivate, ShieldsProtocolPrivate, ToolsProtocolPrivate, + VectorDBsProtocolPrivate, ) log = logging.getLogger(__name__) @@ -62,8 +62,8 @@ def api_protocol_map() -> Dict[Api, Any]: Api.agents: Agents, Api.inference: Inference, Api.inspect: Inspect, - Api.memory: Memory, - Api.memory_banks: MemoryBanks, + Api.vector_io: VectorIO, + Api.vector_dbs: VectorDBs, Api.models: Models, Api.safety: Safety, Api.shields: Shields, @@ -84,7 +84,7 @@ def additional_protocols_map() -> Dict[Api, Any]: return { Api.inference: (ModelsProtocolPrivate, Models, Api.models), Api.tool_groups: (ToolsProtocolPrivate, ToolGroups, Api.tool_groups), - Api.memory: (MemoryBanksProtocolPrivate, MemoryBanks, Api.memory_banks), + Api.vector_io: (VectorDBsProtocolPrivate, VectorDBs, Api.vector_dbs), Api.safety: (ShieldsProtocolPrivate, Shields, Api.shields), Api.datasetio: (DatasetsProtocolPrivate, Datasets, Api.datasets), Api.scoring: ( @@ -145,7 +145,9 @@ async def resolve_impls( log.warning( f"Provider `{provider.provider_type}` for API `{api}` is deprecated and will be removed in a future release: {p.deprecation_warning}", ) - p.deps__ = [a.value for a in p.api_dependencies] + p.deps__ = [a.value for a in p.api_dependencies] + [ + a.value for a in p.optional_api_dependencies + ] spec = ProviderWithSpec( spec=p, **(provider.model_dump()), @@ -229,6 +231,9 @@ async def resolve_impls( inner_impls_by_provider_id = {f"inner-{x.value}": {} for x in router_apis} for api_str, provider in sorted_providers: deps = {a: impls[a] for a in provider.spec.api_dependencies} + for a in provider.spec.optional_api_dependencies: + if a in impls: + deps[a] = impls[a] inner_impls = {} if isinstance(provider.spec, RoutingTableProviderSpec): @@ -265,7 +270,7 @@ def topological_sort( deps.append(dep) for dep in deps: - if dep not in visited: + if dep not in visited and dep in providers_with_specs: dfs((dep, providers_with_specs[dep]), visited, stack) stack.append(api_str) @@ -328,6 +333,8 @@ async def instantiate_provider( impl.__provider_spec__ = provider_spec impl.__provider_config__ = config + # TODO: check compliance for special tool groups + # the impl should be for Api.tool_runtime, the name should be the special tool group, the protocol should be the special tool group protocol check_protocol_compliance(impl, protocols[provider_spec.api]) if ( not isinstance(provider_spec, AutoRoutedProviderSpec) diff --git a/llama_stack/distribution/routers/__init__.py b/llama_stack/distribution/routers/__init__.py index f19a2bffc..156cda385 100644 --- a/llama_stack/distribution/routers/__init__.py +++ b/llama_stack/distribution/routers/__init__.py @@ -14,11 +14,11 @@ from llama_stack.providers.datatypes import Api, RoutingTable from .routing_tables import ( DatasetsRoutingTable, EvalTasksRoutingTable, - MemoryBanksRoutingTable, ModelsRoutingTable, ScoringFunctionsRoutingTable, ShieldsRoutingTable, ToolGroupsRoutingTable, + VectorDBsRoutingTable, ) @@ -29,7 +29,7 @@ async def get_routing_table_impl( dist_registry: DistributionRegistry, ) -> Any: api_to_tables = { - "memory_banks": MemoryBanksRoutingTable, + "vector_dbs": VectorDBsRoutingTable, "models": ModelsRoutingTable, "shields": ShieldsRoutingTable, "datasets": DatasetsRoutingTable, @@ -51,14 +51,14 @@ async def get_auto_router_impl(api: Api, routing_table: RoutingTable, _deps) -> DatasetIORouter, EvalRouter, InferenceRouter, - MemoryRouter, SafetyRouter, ScoringRouter, ToolRuntimeRouter, + VectorIORouter, ) api_to_routers = { - "memory": MemoryRouter, + "vector_io": VectorIORouter, "inference": InferenceRouter, "safety": SafetyRouter, "datasetio": DatasetIORouter, diff --git a/llama_stack/distribution/routers/routers.py b/llama_stack/distribution/routers/routers.py index 8080b9dff..6bb2045bd 100644 --- a/llama_stack/distribution/routers/routers.py +++ b/llama_stack/distribution/routers/routers.py @@ -27,8 +27,6 @@ from llama_stack.apis.inference import ( ToolDefinition, ToolPromptFormat, ) -from llama_stack.apis.memory import Memory, MemoryBankDocument, QueryDocumentsResponse -from llama_stack.apis.memory_banks.memory_banks import BankParams from llama_stack.apis.models import ModelType from llama_stack.apis.safety import RunShieldResponse, Safety from llama_stack.apis.scoring import ( @@ -38,12 +36,20 @@ from llama_stack.apis.scoring import ( ScoringFnParams, ) from llama_stack.apis.shields import Shield -from llama_stack.apis.tools import ToolDef, ToolRuntime +from llama_stack.apis.tools import ( + RAGDocument, + RAGQueryConfig, + RAGQueryResult, + RAGToolRuntime, + ToolDef, + ToolRuntime, +) +from llama_stack.apis.vector_io import Chunk, QueryChunksResponse, VectorIO from llama_stack.providers.datatypes import RoutingTable -class MemoryRouter(Memory): - """Routes to an provider based on the memory bank identifier""" +class VectorIORouter(VectorIO): + """Routes to an provider based on the vector db identifier""" def __init__( self, @@ -57,38 +63,40 @@ class MemoryRouter(Memory): async def shutdown(self) -> None: pass - async def register_memory_bank( + async def register_vector_db( self, - memory_bank_id: str, - params: BankParams, + vector_db_id: str, + embedding_model: str, + embedding_dimension: Optional[int] = 384, provider_id: Optional[str] = None, - provider_memorybank_id: Optional[str] = None, + provider_vector_db_id: Optional[str] = None, ) -> None: - await self.routing_table.register_memory_bank( - memory_bank_id, - params, + await self.routing_table.register_vector_db( + vector_db_id, + embedding_model, + embedding_dimension, provider_id, - provider_memorybank_id, + provider_vector_db_id, ) - async def insert_documents( + async def insert_chunks( self, - bank_id: str, - documents: List[MemoryBankDocument], + vector_db_id: str, + chunks: List[Chunk], ttl_seconds: Optional[int] = None, ) -> None: - return await self.routing_table.get_provider_impl(bank_id).insert_documents( - bank_id, documents, ttl_seconds + return await self.routing_table.get_provider_impl(vector_db_id).insert_chunks( + vector_db_id, chunks, ttl_seconds ) - async def query_documents( + async def query_chunks( self, - bank_id: str, + vector_db_id: str, query: InterleavedContent, params: Optional[Dict[str, Any]] = None, - ) -> QueryDocumentsResponse: - return await self.routing_table.get_provider_impl(bank_id).query_documents( - bank_id, query, params + ) -> QueryChunksResponse: + return await self.routing_table.get_provider_impl(vector_db_id).query_chunks( + vector_db_id, query, params ) @@ -399,22 +407,54 @@ class EvalRouter(Eval): class ToolRuntimeRouter(ToolRuntime): + class RagToolImpl(RAGToolRuntime): + def __init__( + self, + routing_table: RoutingTable, + ) -> None: + self.routing_table = routing_table + + async def query( + self, + content: InterleavedContent, + vector_db_ids: List[str], + query_config: Optional[RAGQueryConfig] = None, + ) -> RAGQueryResult: + return await self.routing_table.get_provider_impl( + "query_from_memory" + ).query(content, vector_db_ids, query_config) + + async def insert( + self, + documents: List[RAGDocument], + vector_db_id: str, + chunk_size_in_tokens: int = 512, + ) -> None: + return await self.routing_table.get_provider_impl( + "insert_into_memory" + ).insert(documents, vector_db_id, chunk_size_in_tokens) + def __init__( self, routing_table: RoutingTable, ) -> None: self.routing_table = routing_table + # HACK ALERT this should be in sync with "get_all_api_endpoints()" + self.rag_tool = self.RagToolImpl(routing_table) + for method in ("query", "insert"): + setattr(self, f"rag_tool.{method}", getattr(self.rag_tool, method)) + async def initialize(self) -> None: pass async def shutdown(self) -> None: pass - async def invoke_tool(self, tool_name: str, args: Dict[str, Any]) -> Any: + async def invoke_tool(self, tool_name: str, kwargs: Dict[str, Any]) -> Any: return await self.routing_table.get_provider_impl(tool_name).invoke_tool( tool_name=tool_name, - args=args, + kwargs=kwargs, ) async def list_runtime_tools( diff --git a/llama_stack/distribution/routers/routing_tables.py b/llama_stack/distribution/routers/routing_tables.py index a3a64bf6b..1d035d878 100644 --- a/llama_stack/distribution/routers/routing_tables.py +++ b/llama_stack/distribution/routers/routing_tables.py @@ -10,23 +10,26 @@ from pydantic import TypeAdapter from llama_stack.apis.common.content_types import URL from llama_stack.apis.common.type_system import ParamType -from llama_stack.apis.datasets import Dataset, Datasets -from llama_stack.apis.eval_tasks import EvalTask, EvalTasks -from llama_stack.apis.memory_banks import ( - BankParams, - MemoryBank, - MemoryBanks, - MemoryBankType, -) -from llama_stack.apis.models import Model, Models, ModelType +from llama_stack.apis.datasets import Dataset, Datasets, ListDatasetsResponse +from llama_stack.apis.eval_tasks import EvalTask, EvalTasks, ListEvalTasksResponse +from llama_stack.apis.models import ListModelsResponse, Model, Models, ModelType from llama_stack.apis.resource import ResourceType from llama_stack.apis.scoring_functions import ( + ListScoringFunctionsResponse, ScoringFn, ScoringFnParams, ScoringFunctions, ) -from llama_stack.apis.shields import Shield, Shields -from llama_stack.apis.tools import Tool, ToolGroup, ToolGroups, ToolHost +from llama_stack.apis.shields import ListShieldsResponse, Shield, Shields +from llama_stack.apis.tools import ( + ListToolGroupsResponse, + ListToolsResponse, + Tool, + ToolGroup, + ToolGroups, + ToolHost, +) +from llama_stack.apis.vector_dbs import ListVectorDBsResponse, VectorDB, VectorDBs from llama_stack.distribution.datatypes import ( RoutableObject, RoutableObjectWithProvider, @@ -50,8 +53,8 @@ async def register_object_with_provider(obj: RoutableObject, p: Any) -> Routable return await p.register_model(obj) elif api == Api.safety: return await p.register_shield(obj) - elif api == Api.memory: - return await p.register_memory_bank(obj) + elif api == Api.vector_io: + return await p.register_vector_db(obj) elif api == Api.datasetio: return await p.register_dataset(obj) elif api == Api.scoring: @@ -66,8 +69,8 @@ async def register_object_with_provider(obj: RoutableObject, p: Any) -> Routable async def unregister_object_from_provider(obj: RoutableObject, p: Any) -> None: api = get_impl_api(p) - if api == Api.memory: - return await p.unregister_memory_bank(obj.identifier) + if api == Api.vector_io: + return await p.unregister_vector_db(obj.identifier) elif api == Api.inference: return await p.unregister_model(obj.identifier) elif api == Api.datasetio: @@ -111,8 +114,8 @@ class CommonRoutingTableImpl(RoutingTable): p.model_store = self elif api == Api.safety: p.shield_store = self - elif api == Api.memory: - p.memory_bank_store = self + elif api == Api.vector_io: + p.vector_db_store = self elif api == Api.datasetio: p.dataset_store = self elif api == Api.scoring: @@ -136,8 +139,8 @@ class CommonRoutingTableImpl(RoutingTable): return ("Inference", "model") elif isinstance(self, ShieldsRoutingTable): return ("Safety", "shield") - elif isinstance(self, MemoryBanksRoutingTable): - return ("Memory", "memory_bank") + elif isinstance(self, VectorDBsRoutingTable): + return ("VectorIO", "vector_db") elif isinstance(self, DatasetsRoutingTable): return ("DatasetIO", "dataset") elif isinstance(self, ScoringFunctionsRoutingTable): @@ -187,9 +190,6 @@ class CommonRoutingTableImpl(RoutingTable): async def register_object( self, obj: RoutableObjectWithProvider ) -> RoutableObjectWithProvider: - # Get existing objects from registry - existing_obj = await self.dist_registry.get(obj.type, obj.identifier) - # if provider_id is not specified, pick an arbitrary one from existing entries if not obj.provider_id and len(self.impls_by_provider_id) > 0: obj.provider_id = list(self.impls_by_provider_id.keys())[0] @@ -215,11 +215,11 @@ class CommonRoutingTableImpl(RoutingTable): class ModelsRoutingTable(CommonRoutingTableImpl, Models): - async def list_models(self) -> List[Model]: - return await self.get_all_with_type("model") + async def list_models(self) -> ListModelsResponse: + return ListModelsResponse(data=await self.get_all_with_type("model")) - async def get_model(self, identifier: str) -> Optional[Model]: - return await self.get_object_by_identifier("model", identifier) + async def get_model(self, model_id: str) -> Optional[Model]: + return await self.get_object_by_identifier("model", model_id) async def register_model( self, @@ -237,7 +237,7 @@ class ModelsRoutingTable(CommonRoutingTableImpl, Models): provider_id = list(self.impls_by_provider_id.keys())[0] else: raise ValueError( - "No provider specified and multiple providers available. Please specify a provider_id. Available providers: {self.impls_by_provider_id.keys()}" + f"No provider specified and multiple providers available. Please specify a provider_id. Available providers: {self.impls_by_provider_id.keys()}" ) if metadata is None: metadata = {} @@ -265,8 +265,10 @@ class ModelsRoutingTable(CommonRoutingTableImpl, Models): class ShieldsRoutingTable(CommonRoutingTableImpl, Shields): - async def list_shields(self) -> List[Shield]: - return await self.get_all_with_type(ResourceType.shield.value) + async def list_shields(self) -> ListShieldsResponse: + return ListShieldsResponse( + data=await self.get_all_with_type(ResourceType.shield.value) + ) async def get_shield(self, identifier: str) -> Optional[Shield]: return await self.get_object_by_identifier("shield", identifier) @@ -300,22 +302,23 @@ class ShieldsRoutingTable(CommonRoutingTableImpl, Shields): return shield -class MemoryBanksRoutingTable(CommonRoutingTableImpl, MemoryBanks): - async def list_memory_banks(self) -> List[MemoryBank]: - return await self.get_all_with_type(ResourceType.memory_bank.value) +class VectorDBsRoutingTable(CommonRoutingTableImpl, VectorDBs): + async def list_vector_dbs(self) -> ListVectorDBsResponse: + return ListVectorDBsResponse(data=await self.get_all_with_type("vector_db")) - async def get_memory_bank(self, memory_bank_id: str) -> Optional[MemoryBank]: - return await self.get_object_by_identifier("memory_bank", memory_bank_id) + async def get_vector_db(self, vector_db_id: str) -> Optional[VectorDB]: + return await self.get_object_by_identifier("vector_db", vector_db_id) - async def register_memory_bank( + async def register_vector_db( self, - memory_bank_id: str, - params: BankParams, + vector_db_id: str, + embedding_model: str, + embedding_dimension: Optional[int] = 384, provider_id: Optional[str] = None, - provider_memory_bank_id: Optional[str] = None, - ) -> MemoryBank: - if provider_memory_bank_id is None: - provider_memory_bank_id = memory_bank_id + provider_vector_db_id: Optional[str] = None, + ) -> VectorDB: + if provider_vector_db_id is None: + provider_vector_db_id = vector_db_id if provider_id is None: # If provider_id not specified, use the only provider if it supports this shield type if len(self.impls_by_provider_id) == 1: @@ -324,49 +327,46 @@ class MemoryBanksRoutingTable(CommonRoutingTableImpl, MemoryBanks): raise ValueError( "No provider specified and multiple providers available. Please specify a provider_id." ) - model = await self.get_object_by_identifier("model", params.embedding_model) + model = await self.get_object_by_identifier("model", embedding_model) if model is None: - if params.embedding_model == "all-MiniLM-L6-v2": + if embedding_model == "all-MiniLM-L6-v2": raise ValueError( "Embeddings are now served via Inference providers. " "Please upgrade your run.yaml to include inline::sentence-transformer as an additional inference provider. " "See https://github.com/meta-llama/llama-stack/blob/main/llama_stack/templates/together/run.yaml for an example." ) else: - raise ValueError(f"Model {params.embedding_model} not found") + raise ValueError(f"Model {embedding_model} not found") if model.model_type != ModelType.embedding: - raise ValueError( - f"Model {params.embedding_model} is not an embedding model" - ) + raise ValueError(f"Model {embedding_model} is not an embedding model") if "embedding_dimension" not in model.metadata: raise ValueError( - f"Model {params.embedding_model} does not have an embedding dimension" + f"Model {embedding_model} does not have an embedding dimension" ) - memory_bank_data = { - "identifier": memory_bank_id, - "type": ResourceType.memory_bank.value, + vector_db_data = { + "identifier": vector_db_id, + "type": ResourceType.vector_db.value, "provider_id": provider_id, - "provider_resource_id": provider_memory_bank_id, - **params.model_dump(), + "provider_resource_id": provider_vector_db_id, + "embedding_model": embedding_model, + "embedding_dimension": model.metadata["embedding_dimension"], } - if params.memory_bank_type == MemoryBankType.vector.value: - memory_bank_data["embedding_dimension"] = model.metadata[ - "embedding_dimension" - ] - memory_bank = TypeAdapter(MemoryBank).validate_python(memory_bank_data) - await self.register_object(memory_bank) - return memory_bank + vector_db = TypeAdapter(VectorDB).validate_python(vector_db_data) + await self.register_object(vector_db) + return vector_db - async def unregister_memory_bank(self, memory_bank_id: str) -> None: - existing_bank = await self.get_memory_bank(memory_bank_id) - if existing_bank is None: - raise ValueError(f"Memory bank {memory_bank_id} not found") - await self.unregister_object(existing_bank) + async def unregister_vector_db(self, vector_db_id: str) -> None: + existing_vector_db = await self.get_vector_db(vector_db_id) + if existing_vector_db is None: + raise ValueError(f"Vector DB {vector_db_id} not found") + await self.unregister_object(existing_vector_db) class DatasetsRoutingTable(CommonRoutingTableImpl, Datasets): - async def list_datasets(self) -> List[Dataset]: - return await self.get_all_with_type(ResourceType.dataset.value) + async def list_datasets(self) -> ListDatasetsResponse: + return ListDatasetsResponse( + data=await self.get_all_with_type(ResourceType.dataset.value) + ) async def get_dataset(self, dataset_id: str) -> Optional[Dataset]: return await self.get_object_by_identifier("dataset", dataset_id) @@ -410,8 +410,10 @@ class DatasetsRoutingTable(CommonRoutingTableImpl, Datasets): class ScoringFunctionsRoutingTable(CommonRoutingTableImpl, ScoringFunctions): - async def list_scoring_functions(self) -> List[ScoringFn]: - return await self.get_all_with_type(ResourceType.scoring_function.value) + async def list_scoring_functions(self) -> ListScoringFunctionsResponse: + return ListScoringFunctionsResponse( + data=await self.get_all_with_type(ResourceType.scoring_function.value) + ) async def get_scoring_function(self, scoring_fn_id: str) -> Optional[ScoringFn]: return await self.get_object_by_identifier("scoring_function", scoring_fn_id) @@ -447,11 +449,11 @@ class ScoringFunctionsRoutingTable(CommonRoutingTableImpl, ScoringFunctions): class EvalTasksRoutingTable(CommonRoutingTableImpl, EvalTasks): - async def list_eval_tasks(self) -> List[EvalTask]: - return await self.get_all_with_type(ResourceType.eval_task.value) + async def list_eval_tasks(self) -> ListEvalTasksResponse: + return ListEvalTasksResponse(data=await self.get_all_with_type("eval_task")) - async def get_eval_task(self, name: str) -> Optional[EvalTask]: - return await self.get_object_by_identifier("eval_task", name) + async def get_eval_task(self, eval_task_id: str) -> Optional[EvalTask]: + return await self.get_object_by_identifier("eval_task", eval_task_id) async def register_eval_task( self, @@ -485,14 +487,14 @@ class EvalTasksRoutingTable(CommonRoutingTableImpl, EvalTasks): class ToolGroupsRoutingTable(CommonRoutingTableImpl, ToolGroups): - async def list_tools(self, tool_group_id: Optional[str] = None) -> List[Tool]: + async def list_tools(self, toolgroup_id: Optional[str] = None) -> ListToolsResponse: tools = await self.get_all_with_type("tool") - if tool_group_id: - tools = [tool for tool in tools if tool.toolgroup_id == tool_group_id] - return tools + if toolgroup_id: + tools = [tool for tool in tools if tool.toolgroup_id == toolgroup_id] + return ListToolsResponse(data=tools) - async def list_tool_groups(self) -> List[ToolGroup]: - return await self.get_all_with_type("tool_group") + async def list_tool_groups(self) -> ListToolGroupsResponse: + return ListToolGroupsResponse(data=await self.get_all_with_type("tool_group")) async def get_tool_group(self, toolgroup_id: str) -> ToolGroup: return await self.get_object_by_identifier("tool_group", toolgroup_id) @@ -551,11 +553,11 @@ class ToolGroupsRoutingTable(CommonRoutingTableImpl, ToolGroups): ) ) - async def unregister_tool_group(self, tool_group_id: str) -> None: - tool_group = await self.get_tool_group(tool_group_id) + async def unregister_toolgroup(self, toolgroup_id: str) -> None: + tool_group = await self.get_tool_group(toolgroup_id) if tool_group is None: - raise ValueError(f"Tool group {tool_group_id} not found") - tools = await self.list_tools(tool_group_id) + raise ValueError(f"Tool group {toolgroup_id} not found") + tools = await self.list_tools(toolgroup_id).data for tool in tools: await self.unregister_object(tool) await self.unregister_object(tool_group) diff --git a/llama_stack/distribution/server/endpoints.py b/llama_stack/distribution/server/endpoints.py index af429e020..180479e40 100644 --- a/llama_stack/distribution/server/endpoints.py +++ b/llama_stack/distribution/server/endpoints.py @@ -9,6 +9,8 @@ from typing import Dict, List from pydantic import BaseModel +from llama_stack.apis.tools import RAGToolRuntime, SpecialToolGroup + from llama_stack.apis.version import LLAMA_STACK_API_VERSION from llama_stack.distribution.resolver import api_protocol_map @@ -22,21 +24,39 @@ class ApiEndpoint(BaseModel): name: str +def toolgroup_protocol_map(): + return { + SpecialToolGroup.rag_tool: RAGToolRuntime, + } + + def get_all_api_endpoints() -> Dict[Api, List[ApiEndpoint]]: apis = {} protocols = api_protocol_map() + toolgroup_protocols = toolgroup_protocol_map() for api, protocol in protocols.items(): endpoints = [] protocol_methods = inspect.getmembers(protocol, predicate=inspect.isfunction) + # HACK ALERT + if api == Api.tool_runtime: + for tool_group in SpecialToolGroup: + sub_protocol = toolgroup_protocols[tool_group] + sub_protocol_methods = inspect.getmembers( + sub_protocol, predicate=inspect.isfunction + ) + for name, method in sub_protocol_methods: + if not hasattr(method, "__webmethod__"): + continue + protocol_methods.append((f"{tool_group.value}.{name}", method)) + for name, method in protocol_methods: if not hasattr(method, "__webmethod__"): continue webmethod = method.__webmethod__ route = f"/{LLAMA_STACK_API_VERSION}/{webmethod.route.lstrip('/')}" - if webmethod.method == "GET": method = "get" elif webmethod.method == "DELETE": diff --git a/llama_stack/distribution/server/server.py b/llama_stack/distribution/server/server.py index 34334de77..8dbb193b9 100644 --- a/llama_stack/distribution/server/server.py +++ b/llama_stack/distribution/server/server.py @@ -14,16 +14,13 @@ import signal import sys import traceback import warnings - from contextlib import asynccontextmanager - from importlib.metadata import version as parse_version from pathlib import Path -from typing import Any, Union +from typing import Any, List, Union import yaml - -from fastapi import Body, FastAPI, HTTPException, Request +from fastapi import Body, FastAPI, HTTPException, Path as FastapiPath, Request from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse, StreamingResponse from pydantic import BaseModel, ValidationError @@ -31,7 +28,6 @@ from termcolor import cprint from typing_extensions import Annotated from llama_stack.distribution.datatypes import StackRunConfig - from llama_stack.distribution.distribution import builtin_automatically_routed_apis from llama_stack.distribution.request_headers import set_request_provider_data from llama_stack.distribution.resolver import InvalidProviderError @@ -41,13 +37,11 @@ from llama_stack.distribution.stack import ( replace_env_vars, validate_env_pair, ) - from llama_stack.providers.datatypes import Api from llama_stack.providers.inline.telemetry.meta_reference.config import TelemetryConfig from llama_stack.providers.inline.telemetry.meta_reference.telemetry import ( TelemetryAdapter, ) - from llama_stack.providers.utils.telemetry.tracing import ( end_trace, setup_logger, @@ -56,7 +50,6 @@ from llama_stack.providers.utils.telemetry.tracing import ( from .endpoints import get_all_api_endpoints - REPO_ROOT = Path(__file__).parent.parent.parent.parent @@ -178,7 +171,7 @@ async def sse_generator(event_gen): ) -def create_dynamic_typed_route(func: Any, method: str): +def create_dynamic_typed_route(func: Any, method: str, route: str): async def endpoint(request: Request, **kwargs): set_request_provider_data(request.headers) @@ -196,6 +189,7 @@ def create_dynamic_typed_route(func: Any, method: str): raise translate_exception(e) from e sig = inspect.signature(func) + new_params = [ inspect.Parameter( "request", inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=Request @@ -203,12 +197,21 @@ def create_dynamic_typed_route(func: Any, method: str): ] new_params.extend(sig.parameters.values()) + path_params = extract_path_params(route) if method == "post": - # make sure every parameter is annotated with Body() so FASTAPI doesn't - # do anything too intelligent and ask for some parameters in the query - # and some in the body + # Annotate parameters that are in the path with Path(...) and others with Body(...) new_params = [new_params[0]] + [ - param.replace(annotation=Annotated[param.annotation, Body(..., embed=True)]) + ( + param.replace( + annotation=Annotated[ + param.annotation, FastapiPath(..., title=param.name) + ] + ) + if param.name in path_params + else param.replace( + annotation=Annotated[param.annotation, Body(..., embed=True)] + ) + ) for param in new_params[1:] ] @@ -260,7 +263,7 @@ class ClientVersionMiddleware: error_msg = json.dumps( { "error": { - "message": f"Client version {client_version} is not compatible with server version {self.server_version}. Please upgrade your client." + "message": f"Client version {client_version} is not compatible with server version {self.server_version}. Please update your client." } } ).encode() @@ -290,7 +293,7 @@ def main(): parser.add_argument( "--port", type=int, - default=int(os.getenv("LLAMA_STACK_PORT", 5000)), + default=int(os.getenv("LLAMA_STACK_PORT", 8321)), help="Port to listen on", ) parser.add_argument( @@ -339,7 +342,8 @@ def main(): app = FastAPI(lifespan=lifespan) app.add_middleware(TracingMiddleware) - app.add_middleware(ClientVersionMiddleware) + if not os.environ.get("LLAMA_STACK_DISABLE_VERSION_CHECK"): + app.add_middleware(ClientVersionMiddleware) try: impls = asyncio.run(construct_stack(config)) @@ -386,6 +390,7 @@ def main(): create_dynamic_typed_route( impl_method, endpoint.method, + endpoint.route, ) ) @@ -409,5 +414,13 @@ def main(): uvicorn.run(app, host=listen_host, port=args.port) +def extract_path_params(route: str) -> List[str]: + segments = route.split("/") + params = [ + seg[1:-1] for seg in segments if seg.startswith("{") and seg.endswith("}") + ] + return params + + if __name__ == "__main__": main() diff --git a/llama_stack/distribution/stack.py b/llama_stack/distribution/stack.py index acbd42fa9..f0c34dba4 100644 --- a/llama_stack/distribution/stack.py +++ b/llama_stack/distribution/stack.py @@ -21,8 +21,6 @@ from llama_stack.apis.eval import Eval from llama_stack.apis.eval_tasks import EvalTasks from llama_stack.apis.inference import Inference from llama_stack.apis.inspect import Inspect -from llama_stack.apis.memory import Memory -from llama_stack.apis.memory_banks import MemoryBanks from llama_stack.apis.models import Models from llama_stack.apis.post_training import PostTraining from llama_stack.apis.safety import Safety @@ -31,7 +29,9 @@ from llama_stack.apis.scoring_functions import ScoringFunctions from llama_stack.apis.shields import Shields from llama_stack.apis.synthetic_data_generation import SyntheticDataGeneration from llama_stack.apis.telemetry import Telemetry -from llama_stack.apis.tools import ToolGroups, ToolRuntime +from llama_stack.apis.tools import RAGToolRuntime, ToolGroups, ToolRuntime +from llama_stack.apis.vector_dbs import VectorDBs +from llama_stack.apis.vector_io import VectorIO from llama_stack.distribution.datatypes import StackRunConfig from llama_stack.distribution.distribution import get_provider_registry from llama_stack.distribution.resolver import ProviderRegistry, resolve_impls @@ -40,11 +40,9 @@ from llama_stack.providers.datatypes import Api log = logging.getLogger(__name__) -LLAMA_STACK_API_VERSION = "alpha" - class LlamaStack( - MemoryBanks, + VectorDBs, Inference, BatchInference, Agents, @@ -53,7 +51,7 @@ class LlamaStack( Datasets, Telemetry, PostTraining, - Memory, + VectorIO, Eval, EvalTasks, Scoring, @@ -64,6 +62,7 @@ class LlamaStack( Inspect, ToolGroups, ToolRuntime, + RAGToolRuntime, ): pass @@ -71,7 +70,7 @@ class LlamaStack( RESOURCES = [ ("models", Api.models, "register_model", "list_models"), ("shields", Api.shields, "register_shield", "list_shields"), - ("memory_banks", Api.memory_banks, "register_memory_bank", "list_memory_banks"), + ("vector_dbs", Api.vector_dbs, "register_vector_db", "list_vector_dbs"), ("datasets", Api.datasets, "register_dataset", "list_datasets"), ( "scoring_fns", @@ -95,7 +94,11 @@ async def register_resources(run_config: StackRunConfig, impls: Dict[Api, Any]): await method(**obj.model_dump()) method = getattr(impls[api], list_method) - for obj in await method(): + response = await method() + + objects_to_process = response.data if hasattr(response, "data") else response + + for obj in objects_to_process: log.info( f"{rsrc.capitalize()}: {colored(obj.identifier, 'white', attrs=['bold'])} served by {colored(obj.provider_id, 'white', attrs=['bold'])}", ) diff --git a/llama_stack/distribution/start_conda_env.sh b/llama_stack/distribution/start_conda_env.sh index f478a8bd8..c37f30ef0 100755 --- a/llama_stack/distribution/start_conda_env.sh +++ b/llama_stack/distribution/start_conda_env.sh @@ -23,8 +23,7 @@ if [ $# -lt 3 ]; then exit 1 fi -build_name="$1" -env_name="llamastack-$build_name" +env_name="$1" shift yaml_config="$1" diff --git a/llama_stack/distribution/start_container.sh b/llama_stack/distribution/start_container.sh index 3b49a22f8..1a55bf96d 100755 --- a/llama_stack/distribution/start_container.sh +++ b/llama_stack/distribution/start_container.sh @@ -6,8 +6,8 @@ # This source code is licensed under the terms described in the LICENSE file in # the root directory of this source tree. -DOCKER_BINARY=${DOCKER_BINARY:-docker} -DOCKER_OPTS=${DOCKER_OPTS:-} +CONTAINER_BINARY=${CONTAINER_BINARY:-docker} +CONTAINER_OPTS=${CONTAINER_OPTS:-} LLAMA_CHECKPOINT_DIR=${LLAMA_CHECKPOINT_DIR:-} LLAMA_STACK_DIR=${LLAMA_STACK_DIR:-} TEST_PYPI_VERSION=${TEST_PYPI_VERSION:-} @@ -31,7 +31,7 @@ if [ $# -lt 3 ]; then fi build_name="$1" -docker_image="localhost/distribution-$build_name" +container_image="localhost/distribution-$build_name" shift yaml_config="$1" @@ -64,7 +64,7 @@ set -x if command -v selinuxenabled &> /dev/null && selinuxenabled; then # Disable SELinux labels - DOCKER_OPTS="$DOCKER_OPTS --security-opt label=disable" + CONTAINER_OPTS="$CONTAINER_OPTS --security-opt label=disable" fi mounts="" @@ -73,7 +73,7 @@ if [ -n "$LLAMA_STACK_DIR" ]; then fi if [ -n "$LLAMA_CHECKPOINT_DIR" ]; then mounts="$mounts -v $LLAMA_CHECKPOINT_DIR:/root/.llama" - DOCKER_OPTS="$DOCKER_OPTS --gpus=all" + CONTAINER_OPTS="$CONTAINER_OPTS --gpus=all" fi version_tag="latest" @@ -85,11 +85,11 @@ elif [ -n "$TEST_PYPI_VERSION" ]; then version_tag="test-$TEST_PYPI_VERSION" fi -$DOCKER_BINARY run $DOCKER_OPTS -it \ +$CONTAINER_BINARY run $CONTAINER_OPTS -it \ -p $port:$port \ $env_vars \ -v "$yaml_config:/app/config.yaml" \ $mounts \ --env LLAMA_STACK_PORT=$port \ --entrypoint='["python", "-m", "llama_stack.distribution.server.server", "--yaml-config", "/app/config.yaml"]' \ - $docker_image:$version_tag + $container_image:$version_tag diff --git a/llama_stack/distribution/store/registry.py b/llama_stack/distribution/store/registry.py index 010d137ec..bf0ff3fd0 100644 --- a/llama_stack/distribution/store/registry.py +++ b/llama_stack/distribution/store/registry.py @@ -35,7 +35,7 @@ class DistributionRegistry(Protocol): REGISTER_PREFIX = "distributions:registry" -KEY_VERSION = "v5" +KEY_VERSION = "v7" KEY_FORMAT = f"{REGISTER_PREFIX}:{KEY_VERSION}::" + "{type}:{identifier}" diff --git a/llama_stack/distribution/store/tests/test_registry.py b/llama_stack/distribution/store/tests/test_registry.py index 54bc04f9c..78d59a088 100644 --- a/llama_stack/distribution/store/tests/test_registry.py +++ b/llama_stack/distribution/store/tests/test_registry.py @@ -9,13 +9,14 @@ import os import pytest import pytest_asyncio from llama_stack.apis.inference import Model -from llama_stack.apis.memory_banks import VectorMemoryBank +from llama_stack.apis.vector_dbs import VectorDB from llama_stack.distribution.store.registry import ( CachedDiskDistributionRegistry, DiskDistributionRegistry, ) -from llama_stack.providers.utils.kvstore import kvstore_impl, SqliteKVStoreConfig +from llama_stack.providers.utils.kvstore import kvstore_impl +from llama_stack.providers.utils.kvstore.config import SqliteKVStoreConfig @pytest.fixture @@ -26,14 +27,14 @@ def config(): return config -@pytest_asyncio.fixture +@pytest_asyncio.fixture(scope="function") async def registry(config): registry = DiskDistributionRegistry(await kvstore_impl(config)) await registry.initialize() return registry -@pytest_asyncio.fixture +@pytest_asyncio.fixture(scope="function") async def cached_registry(config): registry = CachedDiskDistributionRegistry(await kvstore_impl(config)) await registry.initialize() @@ -41,13 +42,12 @@ async def cached_registry(config): @pytest.fixture -def sample_bank(): - return VectorMemoryBank( - identifier="test_bank", +def sample_vector_db(): + return VectorDB( + identifier="test_vector_db", embedding_model="all-MiniLM-L6-v2", - chunk_size_in_tokens=512, - overlap_size_in_tokens=64, - provider_resource_id="test_bank", + embedding_dimension=384, + provider_resource_id="test_vector_db", provider_id="test-provider", ) @@ -64,53 +64,47 @@ def sample_model(): @pytest.mark.asyncio async def test_registry_initialization(registry): # Test empty registry - results = await registry.get("nonexistent", "nonexistent") - assert len(results) == 0 + result = await registry.get("nonexistent", "nonexistent") + assert result is None @pytest.mark.asyncio -async def test_basic_registration(registry, sample_bank, sample_model): - print(f"Registering {sample_bank}") - await registry.register(sample_bank) +async def test_basic_registration(registry, sample_vector_db, sample_model): + print(f"Registering {sample_vector_db}") + await registry.register(sample_vector_db) print(f"Registering {sample_model}") await registry.register(sample_model) - print("Getting bank") - results = await registry.get("memory_bank", "test_bank") - assert len(results) == 1 - result_bank = results[0] - assert result_bank.identifier == sample_bank.identifier - assert result_bank.embedding_model == sample_bank.embedding_model - assert result_bank.chunk_size_in_tokens == sample_bank.chunk_size_in_tokens - assert result_bank.overlap_size_in_tokens == sample_bank.overlap_size_in_tokens - assert result_bank.provider_id == sample_bank.provider_id + print("Getting vector_db") + result_vector_db = await registry.get("vector_db", "test_vector_db") + assert result_vector_db is not None + assert result_vector_db.identifier == sample_vector_db.identifier + assert result_vector_db.embedding_model == sample_vector_db.embedding_model + assert result_vector_db.provider_id == sample_vector_db.provider_id - results = await registry.get("model", "test_model") - assert len(results) == 1 - result_model = results[0] + result_model = await registry.get("model", "test_model") + assert result_model is not None assert result_model.identifier == sample_model.identifier assert result_model.provider_id == sample_model.provider_id @pytest.mark.asyncio -async def test_cached_registry_initialization(config, sample_bank, sample_model): +async def test_cached_registry_initialization(config, sample_vector_db, sample_model): # First populate the disk registry disk_registry = DiskDistributionRegistry(await kvstore_impl(config)) await disk_registry.initialize() - await disk_registry.register(sample_bank) + await disk_registry.register(sample_vector_db) await disk_registry.register(sample_model) # Test cached version loads from disk cached_registry = CachedDiskDistributionRegistry(await kvstore_impl(config)) await cached_registry.initialize() - results = await cached_registry.get("memory_bank", "test_bank") - assert len(results) == 1 - result_bank = results[0] - assert result_bank.identifier == sample_bank.identifier - assert result_bank.embedding_model == sample_bank.embedding_model - assert result_bank.chunk_size_in_tokens == sample_bank.chunk_size_in_tokens - assert result_bank.overlap_size_in_tokens == sample_bank.overlap_size_in_tokens - assert result_bank.provider_id == sample_bank.provider_id + result_vector_db = await cached_registry.get("vector_db", "test_vector_db") + assert result_vector_db is not None + assert result_vector_db.identifier == sample_vector_db.identifier + assert result_vector_db.embedding_model == sample_vector_db.embedding_model + assert result_vector_db.embedding_dimension == sample_vector_db.embedding_dimension + assert result_vector_db.provider_id == sample_vector_db.provider_id @pytest.mark.asyncio @@ -118,31 +112,28 @@ async def test_cached_registry_updates(config): cached_registry = CachedDiskDistributionRegistry(await kvstore_impl(config)) await cached_registry.initialize() - new_bank = VectorMemoryBank( - identifier="test_bank_2", + new_vector_db = VectorDB( + identifier="test_vector_db_2", embedding_model="all-MiniLM-L6-v2", - chunk_size_in_tokens=256, - overlap_size_in_tokens=32, - provider_resource_id="test_bank_2", + embedding_dimension=384, + provider_resource_id="test_vector_db_2", provider_id="baz", ) - await cached_registry.register(new_bank) + await cached_registry.register(new_vector_db) # Verify in cache - results = await cached_registry.get("memory_bank", "test_bank_2") - assert len(results) == 1 - result_bank = results[0] - assert result_bank.identifier == new_bank.identifier - assert result_bank.provider_id == new_bank.provider_id + result_vector_db = await cached_registry.get("vector_db", "test_vector_db_2") + assert result_vector_db is not None + assert result_vector_db.identifier == new_vector_db.identifier + assert result_vector_db.provider_id == new_vector_db.provider_id # Verify persisted to disk new_registry = DiskDistributionRegistry(await kvstore_impl(config)) await new_registry.initialize() - results = await new_registry.get("memory_bank", "test_bank_2") - assert len(results) == 1 - result_bank = results[0] - assert result_bank.identifier == new_bank.identifier - assert result_bank.provider_id == new_bank.provider_id + result_vector_db = await new_registry.get("vector_db", "test_vector_db_2") + assert result_vector_db is not None + assert result_vector_db.identifier == new_vector_db.identifier + assert result_vector_db.provider_id == new_vector_db.provider_id @pytest.mark.asyncio @@ -150,30 +141,28 @@ async def test_duplicate_provider_registration(config): cached_registry = CachedDiskDistributionRegistry(await kvstore_impl(config)) await cached_registry.initialize() - original_bank = VectorMemoryBank( - identifier="test_bank_2", + original_vector_db = VectorDB( + identifier="test_vector_db_2", embedding_model="all-MiniLM-L6-v2", - chunk_size_in_tokens=256, - overlap_size_in_tokens=32, - provider_resource_id="test_bank_2", + embedding_dimension=384, + provider_resource_id="test_vector_db_2", provider_id="baz", ) - await cached_registry.register(original_bank) + await cached_registry.register(original_vector_db) - duplicate_bank = VectorMemoryBank( - identifier="test_bank_2", + duplicate_vector_db = VectorDB( + identifier="test_vector_db_2", embedding_model="different-model", - chunk_size_in_tokens=128, - overlap_size_in_tokens=16, - provider_resource_id="test_bank_2", + embedding_dimension=384, + provider_resource_id="test_vector_db_2", provider_id="baz", # Same provider_id ) - await cached_registry.register(duplicate_bank) + await cached_registry.register(duplicate_vector_db) - results = await cached_registry.get("memory_bank", "test_bank_2") - assert len(results) == 1 # Still only one result + result = await cached_registry.get("vector_db", "test_vector_db_2") + assert result is not None assert ( - results[0].embedding_model == original_bank.embedding_model + result.embedding_model == original_vector_db.embedding_model ) # Original values preserved @@ -183,36 +172,35 @@ async def test_get_all_objects(config): await cached_registry.initialize() # Create multiple test banks - test_banks = [ - VectorMemoryBank( - identifier=f"test_bank_{i}", + test_vector_dbs = [ + VectorDB( + identifier=f"test_vector_db_{i}", embedding_model="all-MiniLM-L6-v2", - chunk_size_in_tokens=256, - overlap_size_in_tokens=32, - provider_resource_id=f"test_bank_{i}", + embedding_dimension=384, + provider_resource_id=f"test_vector_db_{i}", provider_id=f"provider_{i}", ) for i in range(3) ] - # Register all banks - for bank in test_banks: - await cached_registry.register(bank) + # Register all vector_dbs + for vector_db in test_vector_dbs: + await cached_registry.register(vector_db) # Test get_all retrieval all_results = await cached_registry.get_all() assert len(all_results) == 3 - # Verify each bank was stored correctly - for original_bank in test_banks: - matching_banks = [ - b for b in all_results if b.identifier == original_bank.identifier + # Verify each vector_db was stored correctly + for original_vector_db in test_vector_dbs: + matching_vector_dbs = [ + v for v in all_results if v.identifier == original_vector_db.identifier ] - assert len(matching_banks) == 1 - stored_bank = matching_banks[0] - assert stored_bank.embedding_model == original_bank.embedding_model - assert stored_bank.provider_id == original_bank.provider_id - assert stored_bank.chunk_size_in_tokens == original_bank.chunk_size_in_tokens + assert len(matching_vector_dbs) == 1 + stored_vector_db = matching_vector_dbs[0] + assert stored_vector_db.embedding_model == original_vector_db.embedding_model + assert stored_vector_db.provider_id == original_vector_db.provider_id assert ( - stored_bank.overlap_size_in_tokens == original_bank.overlap_size_in_tokens + stored_vector_db.embedding_dimension + == original_vector_db.embedding_dimension ) diff --git a/llama_stack/distribution/tests/library_client_test.py b/llama_stack/distribution/tests/library_client_test.py deleted file mode 100644 index a919ab223..000000000 --- a/llama_stack/distribution/tests/library_client_test.py +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the terms described in the LICENSE file in -# the root directory of this source tree. - -import argparse -import os - -from llama_stack.distribution.library_client import LlamaStackAsLibraryClient -from llama_stack_client.lib.agents.agent import Agent -from llama_stack_client.lib.agents.event_logger import EventLogger as AgentEventLogger -from llama_stack_client.lib.inference.event_logger import EventLogger -from llama_stack_client.types import Attachment, UserMessage -from llama_stack_client.types.agent_create_params import AgentConfig - - -def main(config_path: str): - client = LlamaStackAsLibraryClient(config_path) - if not client.initialize(): - return - - models = client.models.list() - print("\nModels:") - for model in models: - print(model) - - if not models: - print("No models found, skipping chat completion test") - return - - model_id = next(m.identifier for m in models if "8b" in m.identifier.lower()) - print(f"Using model: {model_id}") - response = client.inference.chat_completion( - messages=[UserMessage(content="What is the capital of France?", role="user")], - model_id=model_id, - stream=False, - ) - print("\nChat completion response (non-stream):") - print(response) - - response = client.inference.chat_completion( - messages=[UserMessage(content="What is the capital of France?", role="user")], - model_id=model_id, - stream=True, - ) - - print("\nChat completion response (stream):") - for log in EventLogger().log(response): - log.print() - - print("\nAgent test:") - agent_config = AgentConfig( - model=model_id, - instructions="You are a helpful assistant", - sampling_params={ - "strategy": "greedy", - "temperature": 1.0, - "top_p": 0.9, - }, - tools=( - [ - { - "type": "brave_search", - "engine": "brave", - "api_key": os.getenv("BRAVE_SEARCH_API_KEY"), - } - ] - if os.getenv("BRAVE_SEARCH_API_KEY") - else [] - ) - + ( - [ - { - "type": "code_interpreter", - } - ] - ), - tool_choice="required", - input_shields=[], - output_shields=[], - enable_session_persistence=False, - ) - agent = Agent(client, agent_config) - user_prompts = [ - "Hello", - "Which players played in the winning team of the NBA western conference semifinals of 2024, please use tools", - ] - user_prompts = [ - ( - "Here is a csv, can you describe it ?", - [ - Attachment( - content="https://raw.githubusercontent.com/meta-llama/llama-stack-apps/main/examples/resources/inflation.csv", - mime_type="test/csv", - ) - ], - ), - ("Which year ended with the highest inflation ?", None), - ( - "What macro economic situations that led to such high inflation in that period?", - None, - ), - ("Plot average yearly inflation as a time series", None), - ] - - session_id = agent.create_session("test-session") - - for prompt, attachments in user_prompts: - response = agent.create_turn( - messages=[ - { - "role": "user", - "content": prompt, - } - ], - attachments=attachments, - session_id=session_id, - ) - - for log in AgentEventLogger().log(response): - log.print() - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("config_path", help="Path to the config YAML file") - args = parser.parse_args() - main(args.config_path) diff --git a/llama_stack/distribution/ui/modules/api.py b/llama_stack/distribution/ui/modules/api.py index 807eab19d..7d3367ba5 100644 --- a/llama_stack/distribution/ui/modules/api.py +++ b/llama_stack/distribution/ui/modules/api.py @@ -14,7 +14,7 @@ from llama_stack_client import LlamaStackClient class LlamaStackApi: def __init__(self): self.client = LlamaStackClient( - base_url=os.environ.get("LLAMA_STACK_ENDPOINT", "http://localhost:5000"), + base_url=os.environ.get("LLAMA_STACK_ENDPOINT", "http://localhost:8321"), provider_data={ "fireworks_api_key": os.environ.get("FIREWORKS_API_KEY", ""), "together_api_key": os.environ.get("TOGETHER_API_KEY", ""), diff --git a/llama_stack/distribution/ui/page/distribution/datasets.py b/llama_stack/distribution/ui/page/distribution/datasets.py index 44e314cde..b52356522 100644 --- a/llama_stack/distribution/ui/page/distribution/datasets.py +++ b/llama_stack/distribution/ui/page/distribution/datasets.py @@ -14,6 +14,6 @@ def datasets(): datasets_info = { d.identifier: d.to_dict() for d in llama_stack_api.client.datasets.list() } - - selected_dataset = st.selectbox("Select a dataset", list(datasets_info.keys())) - st.json(datasets_info[selected_dataset], expanded=True) + if len(datasets_info) > 0: + selected_dataset = st.selectbox("Select a dataset", list(datasets_info.keys())) + st.json(datasets_info[selected_dataset], expanded=True) diff --git a/llama_stack/distribution/ui/page/distribution/eval_tasks.py b/llama_stack/distribution/ui/page/distribution/eval_tasks.py index 4957fb178..cc7912838 100644 --- a/llama_stack/distribution/ui/page/distribution/eval_tasks.py +++ b/llama_stack/distribution/ui/page/distribution/eval_tasks.py @@ -16,7 +16,8 @@ def eval_tasks(): d.identifier: d.to_dict() for d in llama_stack_api.client.eval_tasks.list() } - selected_eval_task = st.selectbox( - "Select an eval task", list(eval_tasks_info.keys()), key="eval_task_inspect" - ) - st.json(eval_tasks_info[selected_eval_task], expanded=True) + if len(eval_tasks_info) > 0: + selected_eval_task = st.selectbox( + "Select an eval task", list(eval_tasks_info.keys()), key="eval_task_inspect" + ) + st.json(eval_tasks_info[selected_eval_task], expanded=True) diff --git a/llama_stack/distribution/ui/page/distribution/memory_banks.py b/llama_stack/distribution/ui/page/distribution/memory_banks.py deleted file mode 100644 index f28010bf2..000000000 --- a/llama_stack/distribution/ui/page/distribution/memory_banks.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the terms described in the LICENSE file in -# the root directory of this source tree. - -import streamlit as st -from modules.api import llama_stack_api - - -def memory_banks(): - st.header("Memory Banks") - memory_banks_info = { - m.identifier: m.to_dict() for m in llama_stack_api.client.memory_banks.list() - } - - if len(memory_banks_info) > 0: - selected_memory_bank = st.selectbox( - "Select a memory bank", list(memory_banks_info.keys()) - ) - st.json(memory_banks_info[selected_memory_bank]) - else: - st.info("No memory banks found") diff --git a/llama_stack/distribution/ui/page/distribution/providers.py b/llama_stack/distribution/ui/page/distribution/providers.py index 69f6bd771..9aeb7f2a5 100644 --- a/llama_stack/distribution/ui/page/distribution/providers.py +++ b/llama_stack/distribution/ui/page/distribution/providers.py @@ -10,11 +10,17 @@ from modules.api import llama_stack_api def providers(): st.header("🔍 API Providers") - apis_providers_info = llama_stack_api.client.providers.list() - # selected_api = st.selectbox("Select an API", list(apis_providers_info.keys())) - for api in apis_providers_info.keys(): + apis_providers_lst = llama_stack_api.client.providers.list() + api_to_providers = {} + for api_provider in apis_providers_lst: + if api_provider.api in api_to_providers: + api_to_providers[api_provider.api].append(api_provider) + else: + api_to_providers[api_provider.api] = [api_provider] + + for api in api_to_providers.keys(): st.markdown(f"###### {api}") - st.dataframe([p.to_dict() for p in apis_providers_info[api]], width=500) + st.dataframe([x.to_dict() for x in api_to_providers[api]], width=500) providers() diff --git a/llama_stack/distribution/ui/page/distribution/resources.py b/llama_stack/distribution/ui/page/distribution/resources.py index 6b3ea0e3a..38d494570 100644 --- a/llama_stack/distribution/ui/page/distribution/resources.py +++ b/llama_stack/distribution/ui/page/distribution/resources.py @@ -6,10 +6,10 @@ from page.distribution.datasets import datasets from page.distribution.eval_tasks import eval_tasks -from page.distribution.memory_banks import memory_banks from page.distribution.models import models from page.distribution.scoring_functions import scoring_functions from page.distribution.shields import shields +from page.distribution.vector_dbs import vector_dbs from streamlit_option_menu import option_menu @@ -17,7 +17,7 @@ from streamlit_option_menu import option_menu def resources_page(): options = [ "Models", - "Memory Banks", + "Vector Databases", "Shields", "Scoring Functions", "Datasets", @@ -37,8 +37,8 @@ def resources_page(): ) if selected_resource == "Eval Tasks": eval_tasks() - elif selected_resource == "Memory Banks": - memory_banks() + elif selected_resource == "Vector Databases": + vector_dbs() elif selected_resource == "Datasets": datasets() elif selected_resource == "Models": diff --git a/llama_stack/distribution/ui/page/distribution/vector_dbs.py b/llama_stack/distribution/ui/page/distribution/vector_dbs.py new file mode 100644 index 000000000..9afa6de1f --- /dev/null +++ b/llama_stack/distribution/ui/page/distribution/vector_dbs.py @@ -0,0 +1,23 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +import streamlit as st +from modules.api import llama_stack_api + + +def vector_dbs(): + st.header("Vector Databases") + vector_dbs_info = { + v.identifier: v.to_dict() for v in llama_stack_api.client.vector_dbs.list() + } + + if len(vector_dbs_info) > 0: + selected_vector_db = st.selectbox( + "Select a vector database", list(vector_dbs_info.keys()) + ) + st.json(vector_dbs_info[selected_vector_db]) + else: + st.info("No vector databases found") diff --git a/llama_stack/distribution/ui/page/evaluations/native_eval.py b/llama_stack/distribution/ui/page/evaluations/native_eval.py index 2cbc8d63e..46839e2f9 100644 --- a/llama_stack/distribution/ui/page/evaluations/native_eval.py +++ b/llama_stack/distribution/ui/page/evaluations/native_eval.py @@ -58,11 +58,6 @@ def define_eval_candidate_2(): # Sampling Parameters st.markdown("##### Sampling Parameters") - strategy = st.selectbox( - "Strategy", - ["greedy", "top_p", "top_k"], - index=0, - ) temperature = st.slider( "Temperature", min_value=0.0, @@ -95,13 +90,20 @@ def define_eval_candidate_2(): help="Controls the likelihood for generating the same word or phrase multiple times in the same sentence or paragraph. 1 implies no penalty, 2 will strongly discourage model to repeat words or phrases.", ) if candidate_type == "model": + if temperature > 0.0: + strategy = { + "type": "top_p", + "temperature": temperature, + "top_p": top_p, + } + else: + strategy = {"type": "greedy"} + eval_candidate = { "type": "model", "model": selected_model, "sampling_params": { "strategy": strategy, - "temperature": temperature, - "top_p": top_p, "max_tokens": max_tokens, "repetition_penalty": repetition_penalty, }, diff --git a/llama_stack/distribution/ui/page/playground/chat.py b/llama_stack/distribution/ui/page/playground/chat.py index 0b8073756..cb9990b7c 100644 --- a/llama_stack/distribution/ui/page/playground/chat.py +++ b/llama_stack/distribution/ui/page/playground/chat.py @@ -95,6 +95,15 @@ if prompt := st.chat_input("Example: What is Llama Stack?"): message_placeholder = st.empty() full_response = "" + if temperature > 0.0: + strategy = { + "type": "top_p", + "temperature": temperature, + "top_p": top_p, + } + else: + strategy = {"type": "greedy"} + response = llama_stack_api.client.inference.chat_completion( messages=[ {"role": "system", "content": system_prompt}, @@ -103,8 +112,7 @@ if prompt := st.chat_input("Example: What is Llama Stack?"): model_id=selected_model, stream=stream, sampling_params={ - "temperature": temperature, - "top_p": top_p, + "strategy": strategy, "max_tokens": max_tokens, "repetition_penalty": repetition_penalty, }, @@ -113,7 +121,7 @@ if prompt := st.chat_input("Example: What is Llama Stack?"): if stream: for chunk in response: if chunk.event.event_type == "progress": - full_response += chunk.event.delta + full_response += chunk.event.delta.text message_placeholder.markdown(full_response + "▌") message_placeholder.markdown(full_response) else: diff --git a/llama_stack/distribution/ui/page/playground/rag.py b/llama_stack/distribution/ui/page/playground/rag.py index 196c889ba..49991dc54 100644 --- a/llama_stack/distribution/ui/page/playground/rag.py +++ b/llama_stack/distribution/ui/page/playground/rag.py @@ -29,12 +29,12 @@ def rag_chat_page(): if uploaded_files: st.success(f"Successfully uploaded {len(uploaded_files)} files") # Add memory bank name input field - memory_bank_name = st.text_input( - "Memory Bank Name", - value="rag_bank", - help="Enter a unique identifier for this memory bank", + vector_db_name = st.text_input( + "Vector Database Name", + value="rag_vector_db", + help="Enter a unique identifier for this vector database", ) - if st.button("Create Memory Bank"): + if st.button("Create Vector Database"): documents = [ Document( document_id=uploaded_file.name, @@ -44,34 +44,34 @@ def rag_chat_page(): ] providers = llama_stack_api.client.providers.list() - llama_stack_api.client.memory_banks.register( - memory_bank_id=memory_bank_name, # Use the user-provided name - params={ - "embedding_model": "all-MiniLM-L6-v2", - "chunk_size_in_tokens": 512, - "overlap_size_in_tokens": 64, - }, - provider_id=providers["memory"][0].provider_id, + vector_io_provider = None + + for x in providers: + if x.api == "vector_io": + vector_io_provider = x.provider_id + + llama_stack_api.client.vector_dbs.register( + vector_db_id=vector_db_name, # Use the user-provided name + embedding_dimension=384, + embedding_model="all-MiniLM-L6-v2", + provider_id=vector_io_provider, ) - # insert documents using the custom bank name - llama_stack_api.client.memory.insert( - bank_id=memory_bank_name, # Use the user-provided name + # insert documents using the custom vector db name + llama_stack_api.client.tool_runtime.rag_tool.insert( + vector_db_id=vector_db_name, # Use the user-provided name documents=documents, ) - st.success("Memory bank created successfully!") + st.success("Vector database created successfully!") st.subheader("Configure Agent") # select memory banks - memory_banks = llama_stack_api.client.memory_banks.list() - memory_banks = [bank.identifier for bank in memory_banks] - selected_memory_banks = st.multiselect( - "Select Memory Banks", - memory_banks, + vector_dbs = llama_stack_api.client.vector_dbs.list() + vector_dbs = [vector_db.identifier for vector_db in vector_dbs] + selected_vector_dbs = st.multiselect( + "Select Vector Databases", + vector_dbs, ) - memory_bank_configs = [ - {"bank_id": bank_id, "type": "vector"} for bank_id in selected_memory_banks - ] available_models = llama_stack_api.client.models.list() available_models = [ @@ -118,27 +118,33 @@ def rag_chat_page(): with st.chat_message(message["role"]): st.markdown(message["content"]) + if temperature > 0.0: + strategy = { + "type": "top_p", + "temperature": temperature, + "top_p": top_p, + } + else: + strategy = {"type": "greedy"} + agent_config = AgentConfig( model=selected_model, instructions=system_prompt, sampling_params={ - "strategy": "greedy", - "temperature": temperature, - "top_p": top_p, + "strategy": strategy, }, - tools=[ - { - "type": "memory", - "memory_bank_configs": memory_bank_configs, - "query_generator_config": {"type": "default", "sep": " "}, - "max_tokens_in_context": 4096, - "max_chunks": 10, - } + toolgroups=[ + dict( + name="builtin::rag", + args={ + "vector_db_ids": [ + vector_db_id for vector_db_id in selected_vector_dbs + ], + }, + ) ], tool_choice="auto", tool_prompt_format="json", - input_shields=[], - output_shields=[], enable_session_persistence=False, ) @@ -172,7 +178,7 @@ def rag_chat_page(): retrieval_response = "" for log in EventLogger().log(response): log.print() - if log.role == "memory_retrieval": + if log.role == "tool_execution": retrieval_response += log.content.replace("====", "").strip() retrieval_message_placeholder.info(retrieval_response) else: diff --git a/llama_stack/distribution/utils/exec.py b/llama_stack/distribution/utils/exec.py index 7b06e384d..9b4d0acee 100644 --- a/llama_stack/distribution/utils/exec.py +++ b/llama_stack/distribution/utils/exec.py @@ -98,9 +98,11 @@ def run_with_pty(command): def run_command(command): - process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - output, error = process.communicate() - if process.returncode != 0: - log.error(f"Error: {error.decode('utf-8')}") - sys.exit(1) - return output.decode("utf-8") + try: + result = subprocess.run(command, capture_output=True, text=True, check=True) + print("Script Output\n", result.stdout) + return result.returncode + except subprocess.CalledProcessError as e: + print("Error running script:", e) + print("Error output:", e.stderr) + return e.returncode diff --git a/llama_stack/providers/datatypes.py b/llama_stack/providers/datatypes.py index ce0c9f52e..d0c448f8c 100644 --- a/llama_stack/providers/datatypes.py +++ b/llama_stack/providers/datatypes.py @@ -4,7 +4,6 @@ # This source code is licensed under the terms described in the LICENSE file in # the root directory of this source tree. -from enum import Enum from typing import Any, List, Optional, Protocol from urllib.parse import urlparse @@ -12,38 +11,14 @@ from llama_models.schema_utils import json_schema_type from pydantic import BaseModel, Field from llama_stack.apis.datasets import Dataset + +from llama_stack.apis.datatypes import Api from llama_stack.apis.eval_tasks import EvalTask -from llama_stack.apis.memory_banks.memory_banks import MemoryBank from llama_stack.apis.models import Model from llama_stack.apis.scoring_functions import ScoringFn from llama_stack.apis.shields import Shield from llama_stack.apis.tools import Tool - - -@json_schema_type -class Api(Enum): - inference = "inference" - safety = "safety" - agents = "agents" - memory = "memory" - datasetio = "datasetio" - scoring = "scoring" - eval = "eval" - post_training = "post_training" - tool_runtime = "tool_runtime" - - telemetry = "telemetry" - - models = "models" - shields = "shields" - memory_banks = "memory_banks" - datasets = "datasets" - scoring_functions = "scoring_functions" - eval_tasks = "eval_tasks" - tool_groups = "tool_groups" - - # built-in API - inspect = "inspect" +from llama_stack.apis.vector_dbs import VectorDB class ModelsProtocolPrivate(Protocol): @@ -56,10 +31,10 @@ class ShieldsProtocolPrivate(Protocol): async def register_shield(self, shield: Shield) -> None: ... -class MemoryBanksProtocolPrivate(Protocol): - async def register_memory_bank(self, memory_bank: MemoryBank) -> None: ... +class VectorDBsProtocolPrivate(Protocol): + async def register_vector_db(self, vector_db: VectorDB) -> None: ... - async def unregister_memory_bank(self, memory_bank_id: str) -> None: ... + async def unregister_vector_db(self, vector_db_id: str) -> None: ... class DatasetsProtocolPrivate(Protocol): @@ -96,6 +71,9 @@ class ProviderSpec(BaseModel): default_factory=list, description="Higher-level API surfaces may depend on other providers to provide their functionality", ) + optional_api_dependencies: List[Api] = Field( + default_factory=list, + ) deprecation_warning: Optional[str] = Field( default=None, description="If this provider is deprecated, specify the warning message here", @@ -147,11 +125,11 @@ class InlineProviderSpec(ProviderSpec): default_factory=list, description="The pip dependencies needed for this implementation", ) - docker_image: Optional[str] = Field( + container_image: Optional[str] = Field( default=None, description=""" -The docker image to use for this implementation. If one is provided, pip_packages will be ignored. -If a provider depends on other providers, the dependencies MUST NOT specify a docker image. +The container image to use for this implementation. If one is provided, pip_packages will be ignored. +If a provider depends on other providers, the dependencies MUST NOT specify a container image. """, ) module: str = Field( @@ -194,7 +172,7 @@ API responses, specify the adapter here. ) @property - def docker_image(self) -> Optional[str]: + def container_image(self) -> Optional[str]: return None @property diff --git a/llama_stack/providers/inline/agents/meta_reference/__init__.py b/llama_stack/providers/inline/agents/meta_reference/__init__.py index 50f61fb42..de34b8d2c 100644 --- a/llama_stack/providers/inline/agents/meta_reference/__init__.py +++ b/llama_stack/providers/inline/agents/meta_reference/__init__.py @@ -19,9 +19,8 @@ async def get_provider_impl( impl = MetaReferenceAgentsImpl( config, deps[Api.inference], - deps[Api.memory], + deps[Api.vector_io], deps[Api.safety], - deps[Api.memory_banks], deps[Api.tool_runtime], deps[Api.tool_groups], ) diff --git a/llama_stack/providers/inline/agents/meta_reference/agent_instance.py b/llama_stack/providers/inline/agents/meta_reference/agent_instance.py index 2299e80d1..32801e514 100644 --- a/llama_stack/providers/inline/agents/meta_reference/agent_instance.py +++ b/llama_stack/providers/inline/agents/meta_reference/agent_instance.py @@ -18,6 +18,7 @@ from urllib.parse import urlparse import httpx from llama_models.llama3.api.datatypes import BuiltinTool, ToolCall, ToolParamDefinition +from pydantic import TypeAdapter from llama_stack.apis.agents import ( AgentConfig, @@ -59,13 +60,12 @@ from llama_stack.apis.inference import ( ToolResponseMessage, UserMessage, ) -from llama_stack.apis.memory import Memory, MemoryBankDocument -from llama_stack.apis.memory_banks import MemoryBanks, VectorMemoryBankParams from llama_stack.apis.safety import Safety -from llama_stack.apis.tools import ToolGroups, ToolRuntime +from llama_stack.apis.tools import RAGDocument, RAGQueryConfig, ToolGroups, ToolRuntime +from llama_stack.apis.vector_io import VectorIO from llama_stack.providers.utils.kvstore import KVStore +from llama_stack.providers.utils.memory.vector_store import concat_interleaved_content from llama_stack.providers.utils.telemetry import tracing - from .persistence import AgentPersistence from .safety import SafetyException, ShieldRunnerMixin @@ -79,9 +79,9 @@ def make_random_string(length: int = 8): TOOLS_ATTACHMENT_KEY_REGEX = re.compile(r"__tools_attachment__=(\{.*?\})") -MEMORY_QUERY_TOOL = "query_memory" +MEMORY_QUERY_TOOL = "query_from_memory" WEB_SEARCH_TOOL = "web_search" -MEMORY_GROUP = "builtin::memory" +RAG_TOOL_GROUP = "builtin::rag" class ChatAgent(ShieldRunnerMixin): @@ -91,20 +91,18 @@ class ChatAgent(ShieldRunnerMixin): agent_config: AgentConfig, tempdir: str, inference_api: Inference, - memory_api: Memory, - memory_banks_api: MemoryBanks, safety_api: Safety, tool_runtime_api: ToolRuntime, tool_groups_api: ToolGroups, + vector_io_api: VectorIO, persistence_store: KVStore, ): self.agent_id = agent_id self.agent_config = agent_config self.tempdir = tempdir self.inference_api = inference_api - self.memory_api = memory_api - self.memory_banks_api = memory_banks_api self.safety_api = safety_api + self.vector_io_api = vector_io_api self.storage = AgentPersistence(agent_id, persistence_store) self.tool_runtime_api = tool_runtime_api self.tool_groups_api = tool_groups_api @@ -370,24 +368,30 @@ class ChatAgent(ShieldRunnerMixin): documents: Optional[List[Document]] = None, toolgroups_for_turn: Optional[List[AgentToolGroup]] = None, ) -> AsyncGenerator: + # TODO: simplify all of this code, it can be simpler toolgroup_args = {} + toolgroups = set() for toolgroup in self.agent_config.toolgroups: if isinstance(toolgroup, AgentToolGroupWithArgs): + toolgroups.add(toolgroup.name) toolgroup_args[toolgroup.name] = toolgroup.args + else: + toolgroups.add(toolgroup) if toolgroups_for_turn: for toolgroup in toolgroups_for_turn: if isinstance(toolgroup, AgentToolGroupWithArgs): + toolgroups.add(toolgroup.name) toolgroup_args[toolgroup.name] = toolgroup.args + else: + toolgroups.add(toolgroup) tool_defs, tool_to_group = await self._get_tool_defs(toolgroups_for_turn) if documents: await self.handle_documents( session_id, documents, input_messages, tool_defs ) - if MEMORY_QUERY_TOOL in tool_defs and len(input_messages) > 0: - memory_tool_group = tool_to_group.get(MEMORY_QUERY_TOOL, None) - if memory_tool_group is None: - raise ValueError(f"Memory tool group not found for {MEMORY_QUERY_TOOL}") + + if RAG_TOOL_GROUP in toolgroups and len(input_messages) > 0: with tracing.span(MEMORY_QUERY_TOOL) as span: step_id = str(uuid.uuid4()) yield AgentTurnResponseStreamChunk( @@ -398,17 +402,24 @@ class ChatAgent(ShieldRunnerMixin): ) ) ) - query_args = { - "messages": [msg.content for msg in input_messages], - **toolgroup_args.get(memory_tool_group, {}), - } + + args = toolgroup_args.get(RAG_TOOL_GROUP, {}) + vector_db_ids = args.get("vector_db_ids", []) + query_config = args.get("query_config") + if query_config: + query_config = TypeAdapter(RAGQueryConfig).validate_python( + query_config + ) + else: + # handle someone passing an empty dict + query_config = RAGQueryConfig() session_info = await self.storage.get_session_info(session_id) + # if the session has a memory bank id, let the memory tool use it - if session_info.memory_bank_id: - if "memory_bank_ids" not in query_args: - query_args["memory_bank_ids"] = [] - query_args["memory_bank_ids"].append(session_info.memory_bank_id) + if session_info.vector_db_id: + vector_db_ids.append(session_info.vector_db_id) + yield AgentTurnResponseStreamChunk( event=AgentTurnResponseEvent( payload=AgentTurnResponseStepProgressPayload( @@ -416,7 +427,7 @@ class ChatAgent(ShieldRunnerMixin): step_id=step_id, delta=ToolCallDelta( parse_status=ToolCallParseStatus.succeeded, - content=ToolCall( + tool_call=ToolCall( call_id="", tool_name=MEMORY_QUERY_TOOL, arguments={}, @@ -425,10 +436,14 @@ class ChatAgent(ShieldRunnerMixin): ) ) ) - result = await self.tool_runtime_api.invoke_tool( - tool_name=MEMORY_QUERY_TOOL, - args=query_args, + result = await self.tool_runtime_api.rag_tool.query( + content=concat_interleaved_content( + [msg.content for msg in input_messages] + ), + vector_db_ids=vector_db_ids, + query_config=query_config, ) + retrieved_context = result.content yield AgentTurnResponseStreamChunk( event=AgentTurnResponseEvent( @@ -449,7 +464,7 @@ class ChatAgent(ShieldRunnerMixin): ToolResponse( call_id="", tool_name=MEMORY_QUERY_TOOL, - content=result.content, + content=retrieved_context or [], ) ], ), @@ -459,13 +474,11 @@ class ChatAgent(ShieldRunnerMixin): span.set_attribute( "input", [m.model_dump_json() for m in input_messages] ) - span.set_attribute("output", result.content) - span.set_attribute("error_code", result.error_code) - span.set_attribute("error_message", result.error_message) + span.set_attribute("output", retrieved_context) span.set_attribute("tool_name", MEMORY_QUERY_TOOL) - if result.error_code == 0: + if retrieved_context: last_message = input_messages[-1] - last_message.context = result.content + last_message.context = retrieved_context output_attachments = [] @@ -496,7 +509,7 @@ class ChatAgent(ShieldRunnerMixin): tools=[ tool for tool in tool_defs.values() - if tool_to_group.get(tool.tool_name, None) != MEMORY_GROUP + if tool_to_group.get(tool.tool_name, None) != RAG_TOOL_GROUP ], tool_prompt_format=self.agent_config.tool_prompt_format, stream=True, @@ -512,7 +525,7 @@ class ChatAgent(ShieldRunnerMixin): delta = event.delta if delta.type == "tool_call": if delta.parse_status == ToolCallParseStatus.succeeded: - tool_calls.append(delta.content) + tool_calls.append(delta.tool_call) if stream: yield AgentTurnResponseStreamChunk( event=AgentTurnResponseEvent( @@ -624,6 +637,10 @@ class ChatAgent(ShieldRunnerMixin): step_type=StepType.tool_execution.value, step_id=step_id, tool_call=tool_call, + delta=ToolCallDelta( + parse_status=ToolCallParseStatus.in_progress, + tool_call=tool_call, + ), ) ) ) @@ -735,11 +752,11 @@ class ChatAgent(ShieldRunnerMixin): for toolgroup_name in agent_config_toolgroups: if toolgroup_name not in toolgroups_for_turn_set: continue - tools = await self.tool_groups_api.list_tools(tool_group_id=toolgroup_name) - for tool_def in tools: + tools = await self.tool_groups_api.list_tools(toolgroup_id=toolgroup_name) + for tool_def in tools.data: if ( toolgroup_name.startswith("builtin") - and toolgroup_name != MEMORY_GROUP + and toolgroup_name != RAG_TOOL_GROUP ): tool_name = tool_def.identifier built_in_type = BuiltinTool.brave_search @@ -812,7 +829,7 @@ class ChatAgent(ShieldRunnerMixin): msg = await attachment_message(self.tempdir, url_items) input_messages.append(msg) # Since memory is present, add all the data to the memory bank - await self.add_to_session_memory_bank(session_id, documents) + await self.add_to_session_vector_db(session_id, documents) elif code_interpreter_tool: # if only code_interpreter is available, we download the URLs to a tempdir # and attach the path to them as a message to inference with the @@ -821,7 +838,7 @@ class ChatAgent(ShieldRunnerMixin): input_messages.append(msg) elif memory_tool: # if only memory is available, we load the data from the URLs and content items to the memory bank - await self.add_to_session_memory_bank(session_id, documents) + await self.add_to_session_vector_db(session_id, documents) else: # if no memory or code_interpreter tool is available, # we try to load the data from the URLs and content items as a message to inference @@ -831,32 +848,33 @@ class ChatAgent(ShieldRunnerMixin): + await load_data_from_urls(url_items) ) - async def _ensure_memory_bank(self, session_id: str) -> str: + async def _ensure_vector_db(self, session_id: str) -> str: session_info = await self.storage.get_session_info(session_id) if session_info is None: raise ValueError(f"Session {session_id} not found") - if session_info.memory_bank_id is None: - bank_id = f"memory_bank_{session_id}" - await self.memory_banks_api.register_memory_bank( - memory_bank_id=bank_id, - params=VectorMemoryBankParams( - embedding_model="all-MiniLM-L6-v2", - chunk_size_in_tokens=512, - ), + if session_info.vector_db_id is None: + vector_db_id = f"vector_db_{session_id}" + + # TODO: the semantic for registration is definitely not "creation" + # so we need to fix it if we expect the agent to create a new vector db + # for each session + await self.vector_io_api.register_vector_db( + vector_db_id=vector_db_id, + embedding_model="all-MiniLM-L6-v2", ) - await self.storage.add_memory_bank_to_session(session_id, bank_id) + await self.storage.add_vector_db_to_session(session_id, vector_db_id) else: - bank_id = session_info.memory_bank_id + vector_db_id = session_info.vector_db_id - return bank_id + return vector_db_id - async def add_to_session_memory_bank( + async def add_to_session_vector_db( self, session_id: str, data: List[Document] ) -> None: - bank_id = await self._ensure_memory_bank(session_id) + vector_db_id = await self._ensure_vector_db(session_id) documents = [ - MemoryBankDocument( + RAGDocument( document_id=str(uuid.uuid4()), content=a.content, mime_type=a.mime_type, @@ -864,9 +882,10 @@ class ChatAgent(ShieldRunnerMixin): ) for a in data ] - await self.memory_api.insert_documents( - bank_id=bank_id, + await self.tool_runtime_api.rag_tool.insert( documents=documents, + vector_db_id=vector_db_id, + chunk_size_in_tokens=512, ) @@ -951,7 +970,7 @@ async def execute_tool_call_maybe( result = await tool_runtime_api.invoke_tool( tool_name=name, - args=dict( + kwargs=dict( session_id=session_id, **tool_call_args, ), diff --git a/llama_stack/providers/inline/agents/meta_reference/agents.py b/llama_stack/providers/inline/agents/meta_reference/agents.py index faff716ce..b1844f4d0 100644 --- a/llama_stack/providers/inline/agents/meta_reference/agents.py +++ b/llama_stack/providers/inline/agents/meta_reference/agents.py @@ -26,10 +26,9 @@ from llama_stack.apis.agents import ( Turn, ) from llama_stack.apis.inference import Inference, ToolResponseMessage, UserMessage -from llama_stack.apis.memory import Memory -from llama_stack.apis.memory_banks import MemoryBanks from llama_stack.apis.safety import Safety from llama_stack.apis.tools import ToolGroups, ToolRuntime +from llama_stack.apis.vector_io import VectorIO from llama_stack.providers.utils.kvstore import InmemoryKVStoreImpl, kvstore_impl from .agent_instance import ChatAgent @@ -44,17 +43,15 @@ class MetaReferenceAgentsImpl(Agents): self, config: MetaReferenceAgentsImplConfig, inference_api: Inference, - memory_api: Memory, + vector_io_api: VectorIO, safety_api: Safety, - memory_banks_api: MemoryBanks, tool_runtime_api: ToolRuntime, tool_groups_api: ToolGroups, ): self.config = config self.inference_api = inference_api - self.memory_api = memory_api + self.vector_io_api = vector_io_api self.safety_api = safety_api - self.memory_banks_api = memory_banks_api self.tool_runtime_api = tool_runtime_api self.tool_groups_api = tool_groups_api @@ -114,8 +111,7 @@ class MetaReferenceAgentsImpl(Agents): tempdir=self.tempdir, inference_api=self.inference_api, safety_api=self.safety_api, - memory_api=self.memory_api, - memory_banks_api=self.memory_banks_api, + vector_io_api=self.vector_io_api, tool_runtime_api=self.tool_runtime_api, tool_groups_api=self.tool_groups_api, persistence_store=( @@ -223,5 +219,5 @@ class MetaReferenceAgentsImpl(Agents): async def delete_agents_session(self, agent_id: str, session_id: str) -> None: await self.persistence_store.delete(f"session:{agent_id}:{session_id}") - async def delete_agents(self, agent_id: str) -> None: + async def delete_agent(self, agent_id: str) -> None: await self.persistence_store.delete(f"agent:{agent_id}") diff --git a/llama_stack/providers/inline/agents/meta_reference/persistence.py b/llama_stack/providers/inline/agents/meta_reference/persistence.py index 58b69858b..4b8ad6d4a 100644 --- a/llama_stack/providers/inline/agents/meta_reference/persistence.py +++ b/llama_stack/providers/inline/agents/meta_reference/persistence.py @@ -21,7 +21,7 @@ log = logging.getLogger(__name__) class AgentSessionInfo(BaseModel): session_id: str session_name: str - memory_bank_id: Optional[str] = None + vector_db_id: Optional[str] = None started_at: datetime @@ -52,12 +52,12 @@ class AgentPersistence: return AgentSessionInfo(**json.loads(value)) - async def add_memory_bank_to_session(self, session_id: str, bank_id: str): + async def add_vector_db_to_session(self, session_id: str, vector_db_id: str): session_info = await self.get_session_info(session_id) if session_info is None: raise ValueError(f"Session {session_id} not found") - session_info.memory_bank_id = bank_id + session_info.vector_db_id = vector_db_id await self.kvstore.set( key=f"session:{self.agent_id}:{session_id}", value=session_info.model_dump_json(), diff --git a/llama_stack/providers/inline/agents/meta_reference/tests/test_chat_agent.py b/llama_stack/providers/inline/agents/meta_reference/tests/test_chat_agent.py index a7e6efc8c..09fccd3c6 100644 --- a/llama_stack/providers/inline/agents/meta_reference/tests/test_chat_agent.py +++ b/llama_stack/providers/inline/agents/meta_reference/tests/test_chat_agent.py @@ -29,10 +29,9 @@ from llama_stack.apis.inference import ( SamplingParams, ToolChoice, ToolDefinition, + ToolPromptFormat, UserMessage, ) -from llama_stack.apis.memory import MemoryBank -from llama_stack.apis.memory_banks import BankParams, VectorMemoryBank from llama_stack.apis.safety import RunShieldResponse from llama_stack.apis.tools import ( Tool, @@ -40,8 +39,9 @@ from llama_stack.apis.tools import ( ToolGroup, ToolHost, ToolInvocationResult, - ToolPromptFormat, ) +from llama_stack.apis.vector_io import QueryChunksResponse + from llama_stack.providers.inline.agents.meta_reference.agent_instance import ( MEMORY_QUERY_TOOL, ) @@ -110,68 +110,22 @@ class MockSafetyAPI: return RunShieldResponse(violation=None) -class MockMemoryAPI: +class MockVectorIOAPI: def __init__(self): - self.memory_banks = {} - self.documents = {} + self.chunks = {} - async def create_memory_bank(self, name, config, url=None): - bank_id = f"bank_{len(self.memory_banks)}" - bank = MemoryBank(bank_id, name, config, url) - self.memory_banks[bank_id] = bank - self.documents[bank_id] = {} - return bank + async def insert_chunks(self, vector_db_id, chunks, ttl_seconds=None): + for chunk in chunks: + metadata = chunk.metadata + self.chunks[vector_db_id][metadata["document_id"]] = chunk - async def list_memory_banks(self): - return list(self.memory_banks.values()) + async def query_chunks(self, vector_db_id, query, params=None): + if vector_db_id not in self.chunks: + raise ValueError(f"Bank {vector_db_id} not found") - async def get_memory_bank(self, bank_id): - return self.memory_banks.get(bank_id) - - async def drop_memory_bank(self, bank_id): - if bank_id in self.memory_banks: - del self.memory_banks[bank_id] - del self.documents[bank_id] - return bank_id - - async def insert_documents(self, bank_id, documents, ttl_seconds=None): - if bank_id not in self.documents: - raise ValueError(f"Bank {bank_id} not found") - for doc in documents: - self.documents[bank_id][doc.document_id] = doc - - async def update_documents(self, bank_id, documents): - if bank_id not in self.documents: - raise ValueError(f"Bank {bank_id} not found") - for doc in documents: - if doc.document_id in self.documents[bank_id]: - self.documents[bank_id][doc.document_id] = doc - - async def query_documents(self, bank_id, query, params=None): - if bank_id not in self.documents: - raise ValueError(f"Bank {bank_id} not found") - # Simple mock implementation: return all documents - chunks = [ - {"content": doc.content, "token_count": 10, "document_id": doc.document_id} - for doc in self.documents[bank_id].values() - ] + chunks = list(self.chunks[vector_db_id].values()) scores = [1.0] * len(chunks) - return {"chunks": chunks, "scores": scores} - - async def get_documents(self, bank_id, document_ids): - if bank_id not in self.documents: - raise ValueError(f"Bank {bank_id} not found") - return [ - self.documents[bank_id][doc_id] - for doc_id in document_ids - if doc_id in self.documents[bank_id] - ] - - async def delete_documents(self, bank_id, document_ids): - if bank_id not in self.documents: - raise ValueError(f"Bank {bank_id} not found") - for doc_id in document_ids: - self.documents[bank_id].pop(doc_id, None) + return QueryChunksResponse(chunks=chunks, scores=scores) class MockToolGroupsAPI: @@ -198,7 +152,7 @@ class MockToolGroupsAPI: toolgroup_id=MEMORY_TOOLGROUP, tool_host=ToolHost.client, description="Mock tool", - provider_id="builtin::memory", + provider_id="builtin::rag", parameters=[], ) ] @@ -241,31 +195,6 @@ class MockToolRuntimeAPI: return ToolInvocationResult(content={"result": "Mock tool result"}) -class MockMemoryBanksAPI: - async def list_memory_banks(self) -> List[MemoryBank]: - return [] - - async def get_memory_bank(self, memory_bank_id: str) -> Optional[MemoryBank]: - return None - - async def register_memory_bank( - self, - memory_bank_id: str, - params: BankParams, - provider_id: Optional[str] = None, - provider_memory_bank_id: Optional[str] = None, - ) -> MemoryBank: - return VectorMemoryBank( - identifier=memory_bank_id, - provider_resource_id=provider_memory_bank_id or memory_bank_id, - embedding_model="mock_model", - chunk_size_in_tokens=512, - ) - - async def unregister_memory_bank(self, memory_bank_id: str) -> None: - pass - - @pytest.fixture def mock_inference_api(): return MockInferenceAPI() @@ -277,8 +206,8 @@ def mock_safety_api(): @pytest.fixture -def mock_memory_api(): - return MockMemoryAPI() +def mock_vector_io_api(): + return MockVectorIOAPI() @pytest.fixture @@ -291,17 +220,11 @@ def mock_tool_runtime_api(): return MockToolRuntimeAPI() -@pytest.fixture -def mock_memory_banks_api(): - return MockMemoryBanksAPI() - - @pytest.fixture async def get_agents_impl( mock_inference_api, mock_safety_api, - mock_memory_api, - mock_memory_banks_api, + mock_vector_io_api, mock_tool_runtime_api, mock_tool_groups_api, ): @@ -314,8 +237,7 @@ async def get_agents_impl( ), inference_api=mock_inference_api, safety_api=mock_safety_api, - memory_api=mock_memory_api, - memory_banks_api=mock_memory_banks_api, + vector_io_api=mock_vector_io_api, tool_runtime_api=mock_tool_runtime_api, tool_groups_api=mock_tool_groups_api, ) @@ -338,7 +260,7 @@ async def get_chat_agent(get_agents_impl): return await impl.get_agent(response.agent_id) -MEMORY_TOOLGROUP = "builtin::memory" +MEMORY_TOOLGROUP = "builtin::rag" CODE_INTERPRETER_TOOLGROUP = "builtin::code_interpreter" @@ -484,7 +406,7 @@ async def test_chat_agent_tools( toolgroups_for_turn=[ AgentToolGroupWithArgs( name=MEMORY_TOOLGROUP, - args={"memory_banks": ["test_memory_bank"]}, + args={"vector_dbs": ["test_vector_db"]}, ) ] ) diff --git a/llama_stack/providers/inline/eval/meta_reference/eval.py b/llama_stack/providers/inline/eval/meta_reference/eval.py index 408043db8..63c1e8d98 100644 --- a/llama_stack/providers/inline/eval/meta_reference/eval.py +++ b/llama_stack/providers/inline/eval/meta_reference/eval.py @@ -16,6 +16,9 @@ from llama_stack.apis.scoring import Scoring from llama_stack.distribution.datatypes import Api from llama_stack.providers.datatypes import EvalTasksProtocolPrivate +from llama_stack.providers.inline.agents.meta_reference.agent_instance import ( + MEMORY_QUERY_TOOL, +) from llama_stack.providers.utils.common.data_schema_validator import ( ColumnName, get_valid_schemas, @@ -146,8 +149,12 @@ class MetaReferenceEvalImpl( # check if there's a memory retrieval step and extract the context memory_rag_context = None for step in final_event.turn.steps: - if step.step_type == StepType.memory_retrieval.value: - memory_rag_context = " ".join(x.text for x in step.inserted_context) + if step.step_type == StepType.tool_execution.value: + for tool_response in step.tool_responses: + if tool_response.tool_name == MEMORY_QUERY_TOOL: + memory_rag_context = " ".join( + x.text for x in tool_response.content + ) agent_generation = {} agent_generation[ColumnName.generated_answer.value] = ( diff --git a/llama_stack/providers/inline/inference/meta_reference/generation.py b/llama_stack/providers/inline/inference/meta_reference/generation.py index 1807e4ad5..a96409cab 100644 --- a/llama_stack/providers/inline/inference/meta_reference/generation.py +++ b/llama_stack/providers/inline/inference/meta_reference/generation.py @@ -23,6 +23,11 @@ from fairscale.nn.model_parallel.initialize import ( initialize_model_parallel, model_parallel_is_initialized, ) +from llama_models.datatypes import ( + GreedySamplingStrategy, + SamplingParams, + TopPSamplingStrategy, +) from llama_models.llama3.api.args import ModelArgs from llama_models.llama3.api.chat_format import ChatFormat, LLMInput from llama_models.llama3.api.datatypes import Model @@ -363,11 +368,12 @@ class Llama: max_gen_len = self.model.params.max_seq_len - 1 model_input = self.formatter.encode_content(request.content) + temperature, top_p = _infer_sampling_params(sampling_params) yield from self.generate( model_input=model_input, max_gen_len=max_gen_len, - temperature=sampling_params.temperature, - top_p=sampling_params.top_p, + temperature=temperature, + top_p=top_p, logprobs=bool(request.logprobs), include_stop_token=True, logits_processor=get_logits_processor( @@ -390,14 +396,15 @@ class Llama: ): max_gen_len = self.model.params.max_seq_len - 1 + temperature, top_p = _infer_sampling_params(sampling_params) yield from self.generate( model_input=self.formatter.encode_dialog_prompt( request.messages, request.tool_prompt_format, ), max_gen_len=max_gen_len, - temperature=sampling_params.temperature, - top_p=sampling_params.top_p, + temperature=temperature, + top_p=top_p, logprobs=bool(request.logprobs), include_stop_token=True, logits_processor=get_logits_processor( @@ -492,3 +499,15 @@ def _build_regular_tokens_list( is_word_start_token = len(decoded_after_0) > len(decoded_regular) regular_tokens.append((token_idx, decoded_after_0, is_word_start_token)) return regular_tokens + + +def _infer_sampling_params(sampling_params: SamplingParams): + if isinstance(sampling_params.strategy, GreedySamplingStrategy): + temperature = 0.0 + top_p = 1.0 + elif isinstance(sampling_params.strategy, TopPSamplingStrategy): + temperature = sampling_params.strategy.temperature + top_p = sampling_params.strategy.top_p + else: + raise ValueError(f"Unsupported sampling strategy {sampling_params.strategy}") + return temperature, top_p diff --git a/llama_stack/providers/inline/inference/meta_reference/inference.py b/llama_stack/providers/inline/inference/meta_reference/inference.py index d64d32f03..73962ca7f 100644 --- a/llama_stack/providers/inline/inference/meta_reference/inference.py +++ b/llama_stack/providers/inline/inference/meta_reference/inference.py @@ -193,14 +193,14 @@ class MetaReferenceInferenceImpl( ] yield CompletionResponseStreamChunk( - delta=TextDelta(text=text), + delta=text, stop_reason=stop_reason, logprobs=logprobs if request.logprobs else None, ) if stop_reason is None: yield CompletionResponseStreamChunk( - delta=TextDelta(text=""), + delta="", stop_reason=StopReason.out_of_tokens, ) @@ -223,10 +223,10 @@ class MetaReferenceInferenceImpl( tokenizer = self.generator.formatter.tokenizer for token_result in self.generator.completion(request): tokens.append(token_result.token) - - if token_result.token in tokenizer.stop_tokens: - # not quite right semantically + if token_result.text == "<|eot_id|>": stop_reason = StopReason.end_of_turn + elif token_result.text == "<|eom_id|>": + stop_reason = StopReason.end_of_message if request.logprobs: assert len(token_result.logprobs) == 1 @@ -243,6 +243,10 @@ class MetaReferenceInferenceImpl( stop_reason = StopReason.out_of_tokens content = self.generator.formatter.tokenizer.decode(tokens) + if content.endswith("<|eot_id|>"): + content = content[: -len("<|eot_id|>")] + elif content.endswith("<|eom_id|>"): + content = content[: -len("<|eom_id|>")] return CompletionResponse( content=content, stop_reason=stop_reason, @@ -373,7 +377,7 @@ class MetaReferenceInferenceImpl( event=ChatCompletionResponseEvent( event_type=ChatCompletionResponseEventType.progress, delta=ToolCallDelta( - content="", + tool_call="", parse_status=ToolCallParseStatus.started, ), ) @@ -391,7 +395,7 @@ class MetaReferenceInferenceImpl( if ipython: delta = ToolCallDelta( - content=text, + tool_call=text, parse_status=ToolCallParseStatus.in_progress, ) else: @@ -430,7 +434,7 @@ class MetaReferenceInferenceImpl( event=ChatCompletionResponseEvent( event_type=ChatCompletionResponseEventType.progress, delta=ToolCallDelta( - content="", + tool_call="", parse_status=ToolCallParseStatus.failed, ), stop_reason=stop_reason, @@ -442,7 +446,7 @@ class MetaReferenceInferenceImpl( event=ChatCompletionResponseEvent( event_type=ChatCompletionResponseEventType.progress, delta=ToolCallDelta( - content=tool_call, + tool_call=tool_call, parse_status=ToolCallParseStatus.succeeded, ), stop_reason=stop_reason, diff --git a/llama_stack/providers/inline/inference/meta_reference/parallel_utils.py b/llama_stack/providers/inline/inference/meta_reference/parallel_utils.py index 36720612c..ced712257 100644 --- a/llama_stack/providers/inline/inference/meta_reference/parallel_utils.py +++ b/llama_stack/providers/inline/inference/meta_reference/parallel_utils.py @@ -357,8 +357,8 @@ class ModelParallelProcessGroup: assert not self.running, "inference already running" self.running = True - self.request_socket.send(encode_msg(TaskRequest(task=req))) try: + self.request_socket.send(encode_msg(TaskRequest(task=req))) while True: obj_json = self.request_socket.recv() obj = parse_message(obj_json) diff --git a/llama_stack/providers/inline/inference/meta_reference/quantization/scripts/build_conda.sh b/llama_stack/providers/inline/inference/meta_reference/quantization/scripts/build_conda.sh deleted file mode 100644 index ae0ed0bac..000000000 --- a/llama_stack/providers/inline/inference/meta_reference/quantization/scripts/build_conda.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the terms described in the LICENSE file in -# the root directory of this source tree. - -if [[ $# -ne 1 ]]; then - echo "Error: Please provide the name of CONDA environment you wish to create" - exit 1 -fi - -ENV_NAME=$1 - -set -eu -eval "$(conda shell.bash hook)" - -echo "Will build env (or overwrite) named '$ENV_NAME'" - -set -x - -run_build() { - # Set up the conda environment - yes | conda remove --name $ENV_NAME --all - yes | conda create -n $ENV_NAME python=3.10 - conda activate $ENV_NAME - - # PT nightly - pip install --pre torch --index-url https://download.pytorch.org/whl/nightly/cu121 - - # install dependencies for `llama-agentic-system` - pip install -r fp8_requirements.txt -} - -run_build diff --git a/llama_stack/providers/inline/inference/vllm/vllm.py b/llama_stack/providers/inline/inference/vllm/vllm.py index 0f1045845..49dd8316e 100644 --- a/llama_stack/providers/inline/inference/vllm/vllm.py +++ b/llama_stack/providers/inline/inference/vllm/vllm.py @@ -36,6 +36,7 @@ from llama_stack.apis.inference import ( from llama_stack.apis.models import Model from llama_stack.providers.datatypes import ModelsProtocolPrivate from llama_stack.providers.utils.inference.openai_compat import ( + get_sampling_options, OpenAICompatCompletionChoice, OpenAICompatCompletionResponse, process_chat_completion_response, @@ -126,21 +127,12 @@ class VLLMInferenceImpl(Inference, ModelsProtocolPrivate): if sampling_params is None: return VLLMSamplingParams(max_tokens=self.config.max_tokens) - # TODO convert what I saw in my first test ... but surely there's more to do here - kwargs = { - "temperature": sampling_params.temperature, - "max_tokens": self.config.max_tokens, - } - if sampling_params.top_k: - kwargs["top_k"] = sampling_params.top_k - if sampling_params.top_p: - kwargs["top_p"] = sampling_params.top_p - if sampling_params.max_tokens: - kwargs["max_tokens"] = sampling_params.max_tokens - if sampling_params.repetition_penalty > 0: - kwargs["repetition_penalty"] = sampling_params.repetition_penalty + options = get_sampling_options(sampling_params) + if "repeat_penalty" in options: + options["repetition_penalty"] = options["repeat_penalty"] + del options["repeat_penalty"] - return VLLMSamplingParams(**kwargs) + return VLLMSamplingParams(**options) async def unregister_model(self, model_id: str) -> None: pass diff --git a/llama_stack/providers/inline/memory/__init__.py b/llama_stack/providers/inline/post_training/common/__init__.py similarity index 100% rename from llama_stack/providers/inline/memory/__init__.py rename to llama_stack/providers/inline/post_training/common/__init__.py diff --git a/llama_stack/providers/inline/post_training/common/validator.py b/llama_stack/providers/inline/post_training/common/validator.py new file mode 100644 index 000000000..836e20c85 --- /dev/null +++ b/llama_stack/providers/inline/post_training/common/validator.py @@ -0,0 +1,52 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +# Copyright (c) Meta Platforms, IAny, nc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. +from llama_stack.apis.common.type_system import ( + ChatCompletionInputType, + DialogType, + StringType, +) +from llama_stack.apis.datasets import Datasets +from llama_stack.providers.utils.common.data_schema_validator import ( + ColumnName, + validate_dataset_schema, +) + +EXPECTED_DATASET_SCHEMA = { + "instruct": [ + { + ColumnName.chat_completion_input.value: ChatCompletionInputType(), + ColumnName.expected_answer.value: StringType(), + } + ], + "dialog": [ + { + ColumnName.dialog.value: DialogType(), + } + ], +} + + +async def validate_input_dataset_schema( + datasets_api: Datasets, + dataset_id: str, + dataset_type: str, +) -> None: + dataset_def = await datasets_api.get_dataset(dataset_id=dataset_id) + if not dataset_def.dataset_schema or len(dataset_def.dataset_schema) == 0: + raise ValueError(f"Dataset {dataset_id} does not have a schema defined.") + + if dataset_type not in EXPECTED_DATASET_SCHEMA: + raise ValueError(f"Dataset type {dataset_type} is not supported.") + + validate_dataset_schema( + dataset_def.dataset_schema, EXPECTED_DATASET_SCHEMA[dataset_type] + ) diff --git a/llama_stack/providers/inline/post_training/torchtune/common/utils.py b/llama_stack/providers/inline/post_training/torchtune/common/utils.py index b4cd43770..88011ead4 100644 --- a/llama_stack/providers/inline/post_training/torchtune/common/utils.py +++ b/llama_stack/providers/inline/post_training/torchtune/common/utils.py @@ -10,29 +10,22 @@ # This source code is licensed under the terms described in the LICENSE file in # the root directory of this source tree. -from enum import Enum -from typing import Any, Callable, Dict, List +from typing import Any, Callable, Dict import torch from llama_models.datatypes import Model from llama_models.sku_list import resolve_model from pydantic import BaseModel +from torchtune.data._messages import InputOutputToMessages, ShareGPTToMessages from torchtune.models.llama3 import llama3_tokenizer from torchtune.models.llama3._tokenizer import Llama3Tokenizer from torchtune.models.llama3_1 import lora_llama3_1_8b from torchtune.models.llama3_2 import lora_llama3_2_3b +from torchtune.modules.transforms import Transform -from llama_stack.apis.common.type_system import ParamType, StringType -from llama_stack.apis.datasets import Datasets - - -class ColumnName(Enum): - instruction = "instruction" - input = "input" - output = "output" - text = "text" +from llama_stack.apis.post_training import DatasetFormat class ModelConfig(BaseModel): @@ -41,10 +34,6 @@ class ModelConfig(BaseModel): checkpoint_type: str -class DatasetSchema(BaseModel): - alpaca: List[Dict[str, ParamType]] - - MODEL_CONFIGS: Dict[str, ModelConfig] = { "Llama3.2-3B-Instruct": ModelConfig( model_definition=lora_llama3_2_3b, @@ -58,26 +47,11 @@ MODEL_CONFIGS: Dict[str, ModelConfig] = { ), } +DATA_FORMATS: Dict[str, Transform] = { + "instruct": InputOutputToMessages, + "dialog": ShareGPTToMessages, +} -EXPECTED_DATASET_SCHEMA = DatasetSchema( - alpaca=[ - { - ColumnName.instruction.value: StringType(), - ColumnName.input.value: StringType(), - ColumnName.output.value: StringType(), - ColumnName.text.value: StringType(), - }, - { - ColumnName.instruction.value: StringType(), - ColumnName.input.value: StringType(), - ColumnName.output.value: StringType(), - }, - { - ColumnName.instruction.value: StringType(), - ColumnName.output.value: StringType(), - }, - ] -) BuildLoraModelCallable = Callable[..., torch.nn.Module] BuildTokenizerCallable = Callable[..., Llama3Tokenizer] @@ -124,19 +98,5 @@ async def get_checkpointer_model_type( return model_config.checkpoint_type -async def validate_input_dataset_schema( - datasets_api: Datasets, - dataset_id: str, - dataset_type: str, -) -> None: - dataset_def = await datasets_api.get_dataset(dataset_id=dataset_id) - if not dataset_def.dataset_schema or len(dataset_def.dataset_schema) == 0: - raise ValueError(f"Dataset {dataset_id} does not have a schema defined.") - - if not hasattr(EXPECTED_DATASET_SCHEMA, dataset_type): - raise ValueError(f"Dataset type {dataset_type} is not supported.") - - if dataset_def.dataset_schema not in getattr(EXPECTED_DATASET_SCHEMA, dataset_type): - raise ValueError( - f"Dataset {dataset_id} does not have a correct input schema in {getattr(EXPECTED_DATASET_SCHEMA, dataset_type)}" - ) +async def get_data_transform(data_format: DatasetFormat) -> Transform: + return DATA_FORMATS[data_format.value] diff --git a/llama_stack/providers/inline/post_training/torchtune/datasets/format_adapter.py b/llama_stack/providers/inline/post_training/torchtune/datasets/format_adapter.py new file mode 100644 index 000000000..b4dfbb3c1 --- /dev/null +++ b/llama_stack/providers/inline/post_training/torchtune/datasets/format_adapter.py @@ -0,0 +1,62 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +from typing import Any, Mapping + +from llama_stack.providers.utils.common.data_schema_validator import ColumnName + + +def llama_stack_instruct_to_torchtune_instruct( + sample: Mapping[str, Any] +) -> Mapping[str, Any]: + assert ( + ColumnName.chat_completion_input.value in sample + and ColumnName.expected_answer.value in sample + ), "Invalid input row" + input_messages = eval(str(sample[ColumnName.chat_completion_input.value])) + + assert ( + len(input_messages) == 1 + ), "llama stack intruct dataset format only supports 1 user message" + input_message = input_messages[0] + + assert "content" in input_message, "content not found in input message" + input = input_message["content"] + output = sample[ColumnName.expected_answer.value] + + return { + "input": input, + "output": output, + } + + +def llama_stack_chat_to_torchtune_chat(sample: Mapping[str, Any]) -> Mapping[str, Any]: + assert ColumnName.dialog.value in sample, "Invalid input row" + role_map = {"user": "human", "assistant": "gpt"} + dialog = eval(str(sample[ColumnName.dialog.value])) + + assert len(dialog) > 1, "dialog must have at least 2 messagse" + roles = [] + conversations = [] + for message in dialog: + assert ( + "role" in message and "content" in message + ), "role and content must in message" + roles.append(message["role"]) + conversations.append( + {"from": role_map[message["role"]], "value": message["content"]} + ) + + assert roles[0] == "user", "first message must be from user" + assert "assistant" in roles, "at least 1 message should be from assistant" + + return {"conversations": conversations} diff --git a/llama_stack/providers/inline/post_training/torchtune/datasets/sft.py b/llama_stack/providers/inline/post_training/torchtune/datasets/sft.py index 1f91dc73f..1a5aade09 100644 --- a/llama_stack/providers/inline/post_training/torchtune/datasets/sft.py +++ b/llama_stack/providers/inline/post_training/torchtune/datasets/sft.py @@ -19,6 +19,11 @@ from torchtune.data._common import CROSS_ENTROPY_IGNORE_IDX from torchtune.data._messages import validate_messages from torchtune.modules.transforms import Transform +from llama_stack.providers.inline.post_training.torchtune.datasets.format_adapter import ( + llama_stack_chat_to_torchtune_chat, + llama_stack_instruct_to_torchtune_instruct, +) + class SFTDataset(Dataset): def __init__( @@ -26,10 +31,12 @@ class SFTDataset(Dataset): rows: List[Dict[str, Any]], message_transform: Transform, model_transform: Transform, + dataset_type: str, ) -> None: self._rows = rows self._message_transform = message_transform self._model_transform = model_transform + self._dataset_type = dataset_type def __len__(self): return len(self._rows) @@ -39,6 +46,12 @@ class SFTDataset(Dataset): return self._prepare_sample(sample) def _prepare_sample(self, sample: Mapping[str, Any]) -> Dict[str, Any]: + if self._dataset_type == "instruct": + sample = llama_stack_instruct_to_torchtune_instruct(sample) + elif self._dataset_type == "dialog": + sample = llama_stack_chat_to_torchtune_chat(sample) + else: + raise ValueError(f"Invalid dataset type: {self._dataset_type}") transformed_sample = self._message_transform(sample) if "messages" in transformed_sample: validate_messages(transformed_sample["messages"]) diff --git a/llama_stack/providers/inline/post_training/torchtune/post_training.py b/llama_stack/providers/inline/post_training/torchtune/post_training.py index 90fbf7026..4abe13de2 100644 --- a/llama_stack/providers/inline/post_training/torchtune/post_training.py +++ b/llama_stack/providers/inline/post_training/torchtune/post_training.py @@ -4,7 +4,7 @@ # This source code is licensed under the terms described in the LICENSE file in # the root directory of this source tree. from datetime import datetime -from typing import Any, Dict, List, Optional +from typing import Any, Dict, Optional from llama_models.schema_utils import webmethod @@ -14,6 +14,7 @@ from llama_stack.apis.post_training import ( AlgorithmConfig, DPOAlignmentConfig, JobStatus, + ListPostTrainingJobsResponse, LoraFinetuningConfig, PostTrainingJob, PostTrainingJobArtifactsResponse, @@ -114,8 +115,8 @@ class TorchtunePostTrainingImpl: logger_config: Dict[str, Any], ) -> PostTrainingJob: ... - async def get_training_jobs(self) -> List[PostTrainingJob]: - return self.jobs_list + async def get_training_jobs(self) -> ListPostTrainingJobsResponse: + return ListPostTrainingJobsResponse(data=self.jobs_list) @webmethod(route="/post-training/job/status") async def get_training_job_status( diff --git a/llama_stack/providers/inline/post_training/torchtune/recipes/lora_finetuning_single_device.py b/llama_stack/providers/inline/post_training/torchtune/recipes/lora_finetuning_single_device.py index 6c795d310..80e206ebb 100644 --- a/llama_stack/providers/inline/post_training/torchtune/recipes/lora_finetuning_single_device.py +++ b/llama_stack/providers/inline/post_training/torchtune/recipes/lora_finetuning_single_device.py @@ -4,6 +4,7 @@ # This source code is licensed under the terms described in the LICENSE file in # the root directory of this source tree. +import gc import logging import os import time @@ -18,7 +19,7 @@ from torch import nn from torch.optim import Optimizer from torch.utils.data import DataLoader, DistributedSampler from torchtune import modules, training, utils as torchtune_utils -from torchtune.data import AlpacaToMessages, padded_collate_sft +from torchtune.data import padded_collate_sft from torchtune.modules.loss import CEWithChunkedOutputLoss from torchtune.modules.peft import ( @@ -47,6 +48,9 @@ from llama_stack.apis.post_training import ( from llama_stack.distribution.utils.config_dirs import DEFAULT_CHECKPOINT_DIR from llama_stack.distribution.utils.model_utils import model_local_dir +from llama_stack.providers.inline.post_training.common.validator import ( + validate_input_dataset_schema, +) from llama_stack.providers.inline.post_training.torchtune.common import utils from llama_stack.providers.inline.post_training.torchtune.common.checkpointer import ( @@ -129,8 +133,10 @@ class LoraFinetuningSingleDevice: self.seed = training.set_seed(seed=config.torch_seed) self.epochs_run = 0 self.total_epochs = training_config.n_epochs + self._data_format = training_config.data_config.data_format self._shuffle = training_config.data_config.shuffle self._batch_size = training_config.data_config.batch_size + self._train_on_input = training_config.data_config.train_on_input # this is important for debugging purpose self.max_steps_per_epoch = training_config.max_steps_per_epoch @@ -354,18 +360,17 @@ class LoraFinetuningSingleDevice: all_rows = await fetch_rows(dataset_id) rows = all_rows.rows - # Curretly only support alpaca instruct dataset - # TODO @SLR722 make the message_transform swappable and support more dataset types - # TODO @SLR722 make the input dataset schema more flexible by exposing column_map - await utils.validate_input_dataset_schema( + await validate_input_dataset_schema( datasets_api=self.datasets_api, dataset_id=dataset_id, - dataset_type="alpaca", + dataset_type=self._data_format.value, ) + data_transform = await utils.get_data_transform(self._data_format) ds = SFTDataset( rows, - message_transform=AlpacaToMessages(train_on_input=False), + message_transform=data_transform(train_on_input=self._train_on_input), model_transform=tokenizer, + dataset_type=self._data_format.value, ) sampler = DistributedSampler( @@ -576,6 +581,12 @@ class LoraFinetuningSingleDevice: checkpoint.training_metrics = training_metrics checkpoints.append(checkpoint) + # clean up the memory after training finishes + self._model.to("cpu") + del self._model + gc.collect() + torch.cuda.empty_cache() + return (memory_stats, checkpoints) async def validation(self) -> Tuple[float, float]: diff --git a/llama_stack/providers/inline/safety/llama_guard/llama_guard.py b/llama_stack/providers/inline/safety/llama_guard/llama_guard.py index 00213ac83..bc4d9640c 100644 --- a/llama_stack/providers/inline/safety/llama_guard/llama_guard.py +++ b/llama_stack/providers/inline/safety/llama_guard/llama_guard.py @@ -263,9 +263,11 @@ class LlamaGuardShield: stream=True, ): event = chunk.event - if event.event_type == ChatCompletionResponseEventType.progress: - assert isinstance(event.delta, str) - content += event.delta + if ( + event.event_type == ChatCompletionResponseEventType.progress + and event.delta.type == "text" + ): + content += event.delta.text content = content.strip() return self.get_shield_response(content) diff --git a/llama_stack/providers/inline/telemetry/meta_reference/telemetry.py b/llama_stack/providers/inline/telemetry/meta_reference/telemetry.py index efc37b553..aeeed1ac0 100644 --- a/llama_stack/providers/inline/telemetry/meta_reference/telemetry.py +++ b/llama_stack/providers/inline/telemetry/meta_reference/telemetry.py @@ -21,22 +21,21 @@ from llama_stack.apis.telemetry import ( Event, MetricEvent, QueryCondition, + QuerySpanTreeResponse, + QueryTracesResponse, + Span, SpanEndPayload, SpanStartPayload, SpanStatus, - SpanWithStatus, StructuredLogEvent, Telemetry, Trace, UnstructuredLogEvent, ) - from llama_stack.distribution.datatypes import Api - from llama_stack.providers.inline.telemetry.meta_reference.console_span_processor import ( ConsoleSpanProcessor, ) - from llama_stack.providers.inline.telemetry.meta_reference.sqlite_span_processor import ( SQLiteSpanProcessor, ) @@ -52,6 +51,7 @@ _GLOBAL_STORAGE = { "up_down_counters": {}, } _global_lock = threading.Lock() +_TRACER_PROVIDER = None def string_to_trace_id(s: str) -> int: @@ -72,7 +72,7 @@ def is_tracing_enabled(tracer): class TelemetryAdapter(TelemetryDatasetMixin, Telemetry): def __init__(self, config: TelemetryConfig, deps: Dict[str, Any]) -> None: self.config = config - self.datasetio_api = deps[Api.datasetio] + self.datasetio_api = deps.get(Api.datasetio) resource = Resource.create( { @@ -80,31 +80,34 @@ class TelemetryAdapter(TelemetryDatasetMixin, Telemetry): } ) - provider = TracerProvider(resource=resource) - trace.set_tracer_provider(provider) - if TelemetrySink.OTEL in self.config.sinks: - otlp_exporter = OTLPSpanExporter( - endpoint=self.config.otel_endpoint, - ) - span_processor = BatchSpanProcessor(otlp_exporter) - trace.get_tracer_provider().add_span_processor(span_processor) - metric_reader = PeriodicExportingMetricReader( - OTLPMetricExporter( + global _TRACER_PROVIDER + if _TRACER_PROVIDER is None: + provider = TracerProvider(resource=resource) + trace.set_tracer_provider(provider) + _TRACER_PROVIDER = provider + if TelemetrySink.OTEL in self.config.sinks: + otlp_exporter = OTLPSpanExporter( endpoint=self.config.otel_endpoint, ) - ) - metric_provider = MeterProvider( - resource=resource, metric_readers=[metric_reader] - ) - metrics.set_meter_provider(metric_provider) - self.meter = metrics.get_meter(__name__) - if TelemetrySink.SQLITE in self.config.sinks: - trace.get_tracer_provider().add_span_processor( - SQLiteSpanProcessor(self.config.sqlite_db_path) - ) - self.trace_store = SQLiteTraceStore(self.config.sqlite_db_path) - if TelemetrySink.CONSOLE in self.config.sinks: - trace.get_tracer_provider().add_span_processor(ConsoleSpanProcessor()) + span_processor = BatchSpanProcessor(otlp_exporter) + trace.get_tracer_provider().add_span_processor(span_processor) + metric_reader = PeriodicExportingMetricReader( + OTLPMetricExporter( + endpoint=self.config.otel_endpoint, + ) + ) + metric_provider = MeterProvider( + resource=resource, metric_readers=[metric_reader] + ) + metrics.set_meter_provider(metric_provider) + self.meter = metrics.get_meter(__name__) + if TelemetrySink.SQLITE in self.config.sinks: + trace.get_tracer_provider().add_span_processor( + SQLiteSpanProcessor(self.config.sqlite_db_path) + ) + self.trace_store = SQLiteTraceStore(self.config.sqlite_db_path) + if TelemetrySink.CONSOLE in self.config.sinks: + trace.get_tracer_provider().add_span_processor(ConsoleSpanProcessor()) self._lock = _global_lock async def initialize(self) -> None: @@ -240,22 +243,32 @@ class TelemetryAdapter(TelemetryDatasetMixin, Telemetry): limit: Optional[int] = 100, offset: Optional[int] = 0, order_by: Optional[List[str]] = None, - ) -> List[Trace]: - return await self.trace_store.query_traces( - attribute_filters=attribute_filters, - limit=limit, - offset=offset, - order_by=order_by, + ) -> QueryTracesResponse: + return QueryTracesResponse( + data=await self.trace_store.query_traces( + attribute_filters=attribute_filters, + limit=limit, + offset=offset, + order_by=order_by, + ) ) + async def get_trace(self, trace_id: str) -> Trace: + return await self.trace_store.get_trace(trace_id) + + async def get_span(self, trace_id: str, span_id: str) -> Span: + return await self.trace_store.get_span(trace_id, span_id) + async def get_span_tree( self, span_id: str, attributes_to_return: Optional[List[str]] = None, max_depth: Optional[int] = None, - ) -> Dict[str, SpanWithStatus]: - return await self.trace_store.get_span_tree( - span_id=span_id, - attributes_to_return=attributes_to_return, - max_depth=max_depth, + ) -> QuerySpanTreeResponse: + return QuerySpanTreeResponse( + data=await self.trace_store.get_span_tree( + span_id=span_id, + attributes_to_return=attributes_to_return, + max_depth=max_depth, + ) ) diff --git a/llama_stack/providers/inline/tool_runtime/code_interpreter/code_interpreter.py b/llama_stack/providers/inline/tool_runtime/code_interpreter/code_interpreter.py index 361c91a92..04434768d 100644 --- a/llama_stack/providers/inline/tool_runtime/code_interpreter/code_interpreter.py +++ b/llama_stack/providers/inline/tool_runtime/code_interpreter/code_interpreter.py @@ -60,9 +60,9 @@ class CodeInterpreterToolRuntimeImpl(ToolsProtocolPrivate, ToolRuntime): ] async def invoke_tool( - self, tool_name: str, args: Dict[str, Any] + self, tool_name: str, kwargs: Dict[str, Any] ) -> ToolInvocationResult: - script = args["code"] + script = kwargs["code"] req = CodeExecutionRequest(scripts=[script]) res = self.code_executor.execute(req) pieces = [res["process_status"]] diff --git a/llama_stack/providers/inline/tool_runtime/memory/config.py b/llama_stack/providers/inline/tool_runtime/memory/config.py deleted file mode 100644 index 6ff242c6b..000000000 --- a/llama_stack/providers/inline/tool_runtime/memory/config.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the terms described in the LICENSE file in -# the root directory of this source tree. - -from enum import Enum -from typing import Annotated, List, Literal, Union - -from pydantic import BaseModel, Field - - -class _MemoryBankConfigCommon(BaseModel): - bank_id: str - - -class VectorMemoryBankConfig(_MemoryBankConfigCommon): - type: Literal["vector"] = "vector" - - -class KeyValueMemoryBankConfig(_MemoryBankConfigCommon): - type: Literal["keyvalue"] = "keyvalue" - keys: List[str] # what keys to focus on - - -class KeywordMemoryBankConfig(_MemoryBankConfigCommon): - type: Literal["keyword"] = "keyword" - - -class GraphMemoryBankConfig(_MemoryBankConfigCommon): - type: Literal["graph"] = "graph" - entities: List[str] # what entities to focus on - - -MemoryBankConfig = Annotated[ - Union[ - VectorMemoryBankConfig, - KeyValueMemoryBankConfig, - KeywordMemoryBankConfig, - GraphMemoryBankConfig, - ], - Field(discriminator="type"), -] - - -class MemoryQueryGenerator(Enum): - default = "default" - llm = "llm" - custom = "custom" - - -class DefaultMemoryQueryGeneratorConfig(BaseModel): - type: Literal[MemoryQueryGenerator.default.value] = ( - MemoryQueryGenerator.default.value - ) - sep: str = " " - - -class LLMMemoryQueryGeneratorConfig(BaseModel): - type: Literal[MemoryQueryGenerator.llm.value] = MemoryQueryGenerator.llm.value - model: str - template: str - - -class CustomMemoryQueryGeneratorConfig(BaseModel): - type: Literal[MemoryQueryGenerator.custom.value] = MemoryQueryGenerator.custom.value - - -MemoryQueryGeneratorConfig = Annotated[ - Union[ - DefaultMemoryQueryGeneratorConfig, - LLMMemoryQueryGeneratorConfig, - CustomMemoryQueryGeneratorConfig, - ], - Field(discriminator="type"), -] - - -class MemoryToolConfig(BaseModel): - memory_bank_configs: List[MemoryBankConfig] = Field(default_factory=list) - - -class MemoryToolRuntimeConfig(BaseModel): - # This config defines how a query is generated using the messages - # for memory bank retrieval. - query_generator_config: MemoryQueryGeneratorConfig = Field( - default=DefaultMemoryQueryGeneratorConfig() - ) - max_tokens_in_context: int = 4096 - max_chunks: int = 5 diff --git a/llama_stack/providers/inline/tool_runtime/memory/memory.py b/llama_stack/providers/inline/tool_runtime/memory/memory.py deleted file mode 100644 index fe6325abb..000000000 --- a/llama_stack/providers/inline/tool_runtime/memory/memory.py +++ /dev/null @@ -1,146 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the terms described in the LICENSE file in -# the root directory of this source tree. - -import asyncio -import logging -import secrets -import string -from typing import Any, Dict, List, Optional - -from llama_stack.apis.common.content_types import URL -from llama_stack.apis.inference import Inference, InterleavedContent -from llama_stack.apis.memory import Memory, QueryDocumentsResponse -from llama_stack.apis.memory_banks import MemoryBanks -from llama_stack.apis.tools import ( - ToolDef, - ToolInvocationResult, - ToolParameter, - ToolRuntime, -) -from llama_stack.providers.datatypes import ToolsProtocolPrivate -from llama_stack.providers.utils.memory.vector_store import concat_interleaved_content - -from .config import MemoryToolConfig, MemoryToolRuntimeConfig -from .context_retriever import generate_rag_query - -log = logging.getLogger(__name__) - - -def make_random_string(length: int = 8): - return "".join( - secrets.choice(string.ascii_letters + string.digits) for _ in range(length) - ) - - -class MemoryToolRuntimeImpl(ToolsProtocolPrivate, ToolRuntime): - def __init__( - self, - config: MemoryToolRuntimeConfig, - memory_api: Memory, - memory_banks_api: MemoryBanks, - inference_api: Inference, - ): - self.config = config - self.memory_api = memory_api - self.memory_banks_api = memory_banks_api - self.inference_api = inference_api - - async def initialize(self): - pass - - async def list_runtime_tools( - self, tool_group_id: Optional[str] = None, mcp_endpoint: Optional[URL] = None - ) -> List[ToolDef]: - return [ - ToolDef( - name="query_memory", - description="Retrieve context from memory", - parameters=[ - ToolParameter( - name="messages", - description="The input messages to search for", - parameter_type="array", - ), - ], - ) - ] - - async def _retrieve_context( - self, input_messages: List[InterleavedContent], bank_ids: List[str] - ) -> Optional[List[InterleavedContent]]: - if not bank_ids: - return None - query = await generate_rag_query( - self.config.query_generator_config, - input_messages, - inference_api=self.inference_api, - ) - tasks = [ - self.memory_api.query_documents( - bank_id=bank_id, - query=query, - params={ - "max_chunks": self.config.max_chunks, - }, - ) - for bank_id in bank_ids - ] - results: List[QueryDocumentsResponse] = await asyncio.gather(*tasks) - chunks = [c for r in results for c in r.chunks] - scores = [s for r in results for s in r.scores] - - if not chunks: - return None - - # sort by score - chunks, scores = zip( - *sorted(zip(chunks, scores), key=lambda x: x[1], reverse=True) - ) - - tokens = 0 - picked = [] - for c in chunks[: self.config.max_chunks]: - tokens += c.token_count - if tokens > self.config.max_tokens_in_context: - log.error( - f"Using {len(picked)} chunks; reached max tokens in context: {tokens}", - ) - break - picked.append(f"id:{c.document_id}; content:{c.content}") - - return [ - "Here are the retrieved documents for relevant context:\n=== START-RETRIEVED-CONTEXT ===\n", - *picked, - "\n=== END-RETRIEVED-CONTEXT ===\n", - ] - - async def invoke_tool( - self, tool_name: str, args: Dict[str, Any] - ) -> ToolInvocationResult: - tool = await self.tool_store.get_tool(tool_name) - tool_group = await self.tool_store.get_tool_group(tool.toolgroup_id) - final_args = tool_group.args or {} - final_args.update(args) - config = MemoryToolConfig() - if tool.metadata and tool.metadata.get("config") is not None: - config = MemoryToolConfig(**tool.metadata["config"]) - if "memory_bank_ids" in final_args: - bank_ids = final_args["memory_bank_ids"] - else: - bank_ids = [ - bank_config.bank_id for bank_config in config.memory_bank_configs - ] - if "messages" not in final_args: - raise ValueError("messages are required") - context = await self._retrieve_context( - final_args["messages"], - bank_ids, - ) - if context is None: - context = [] - return ToolInvocationResult( - content=concat_interleaved_content(context), error_code=0 - ) diff --git a/llama_stack/providers/inline/tool_runtime/memory/__init__.py b/llama_stack/providers/inline/tool_runtime/rag/__init__.py similarity index 59% rename from llama_stack/providers/inline/tool_runtime/memory/__init__.py rename to llama_stack/providers/inline/tool_runtime/rag/__init__.py index 928afa484..542872091 100644 --- a/llama_stack/providers/inline/tool_runtime/memory/__init__.py +++ b/llama_stack/providers/inline/tool_runtime/rag/__init__.py @@ -8,13 +8,11 @@ from typing import Any, Dict from llama_stack.providers.datatypes import Api -from .config import MemoryToolRuntimeConfig +from .config import RagToolRuntimeConfig from .memory import MemoryToolRuntimeImpl -async def get_provider_impl(config: MemoryToolRuntimeConfig, deps: Dict[str, Any]): - impl = MemoryToolRuntimeImpl( - config, deps[Api.memory], deps[Api.memory_banks], deps[Api.inference] - ) +async def get_provider_impl(config: RagToolRuntimeConfig, deps: Dict[str, Any]): + impl = MemoryToolRuntimeImpl(config, deps[Api.vector_io], deps[Api.inference]) await impl.initialize() return impl diff --git a/llama_stack/providers/inline/tool_runtime/rag/config.py b/llama_stack/providers/inline/tool_runtime/rag/config.py new file mode 100644 index 000000000..2d0d2f595 --- /dev/null +++ b/llama_stack/providers/inline/tool_runtime/rag/config.py @@ -0,0 +1,11 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +from pydantic import BaseModel + + +class RagToolRuntimeConfig(BaseModel): + pass diff --git a/llama_stack/providers/inline/tool_runtime/memory/context_retriever.py b/llama_stack/providers/inline/tool_runtime/rag/context_retriever.py similarity index 55% rename from llama_stack/providers/inline/tool_runtime/memory/context_retriever.py rename to llama_stack/providers/inline/tool_runtime/rag/context_retriever.py index 803981f07..e77ec76af 100644 --- a/llama_stack/providers/inline/tool_runtime/memory/context_retriever.py +++ b/llama_stack/providers/inline/tool_runtime/rag/context_retriever.py @@ -5,68 +5,64 @@ # the root directory of this source tree. -from typing import List - from jinja2 import Template -from pydantic import BaseModel from llama_stack.apis.common.content_types import InterleavedContent from llama_stack.apis.inference import UserMessage + +from llama_stack.apis.tools.rag_tool import ( + DefaultRAGQueryGeneratorConfig, + LLMRAGQueryGeneratorConfig, + RAGQueryGenerator, + RAGQueryGeneratorConfig, +) from llama_stack.providers.utils.inference.prompt_adapter import ( interleaved_content_as_str, ) -from .config import ( - DefaultMemoryQueryGeneratorConfig, - LLMMemoryQueryGeneratorConfig, - MemoryQueryGenerator, - MemoryQueryGeneratorConfig, -) - async def generate_rag_query( - config: MemoryQueryGeneratorConfig, - messages: List[InterleavedContent], + config: RAGQueryGeneratorConfig, + content: InterleavedContent, **kwargs, ): """ Generates a query that will be used for retrieving relevant information from the memory bank. """ - if config.type == MemoryQueryGenerator.default.value: - query = await default_rag_query_generator(config, messages, **kwargs) - elif config.type == MemoryQueryGenerator.llm.value: - query = await llm_rag_query_generator(config, messages, **kwargs) + if config.type == RAGQueryGenerator.default.value: + query = await default_rag_query_generator(config, content, **kwargs) + elif config.type == RAGQueryGenerator.llm.value: + query = await llm_rag_query_generator(config, content, **kwargs) else: raise NotImplementedError(f"Unsupported memory query generator {config.type}") return query async def default_rag_query_generator( - config: DefaultMemoryQueryGeneratorConfig, - messages: List[InterleavedContent], + config: DefaultRAGQueryGeneratorConfig, + content: InterleavedContent, **kwargs, ): - return config.sep.join(interleaved_content_as_str(m) for m in messages) + return interleaved_content_as_str(content, sep=config.separator) async def llm_rag_query_generator( - config: LLMMemoryQueryGeneratorConfig, - messages: List[InterleavedContent], + config: LLMRAGQueryGeneratorConfig, + content: InterleavedContent, **kwargs, ): assert "inference_api" in kwargs, "LLMRAGQueryGenerator needs inference_api" inference_api = kwargs["inference_api"] - m_dict = { - "messages": [ - message.model_dump() if isinstance(message, BaseModel) else message - for message in messages - ] - } + messages = [] + if isinstance(content, list): + messages = [interleaved_content_as_str(m) for m in content] + else: + messages = [interleaved_content_as_str(content)] template = Template(config.template) - content = template.render(m_dict) + content = template.render({"messages": messages}) model = config.model message = UserMessage(content=content) diff --git a/llama_stack/providers/inline/tool_runtime/rag/memory.py b/llama_stack/providers/inline/tool_runtime/rag/memory.py new file mode 100644 index 000000000..9a2687925 --- /dev/null +++ b/llama_stack/providers/inline/tool_runtime/rag/memory.py @@ -0,0 +1,177 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +import asyncio +import logging +import secrets +import string +from typing import Any, Dict, List, Optional + +from llama_stack.apis.common.content_types import ( + InterleavedContent, + TextContentItem, + URL, +) +from llama_stack.apis.inference import Inference +from llama_stack.apis.tools import ( + RAGDocument, + RAGQueryConfig, + RAGQueryResult, + RAGToolRuntime, + ToolDef, + ToolInvocationResult, + ToolRuntime, +) +from llama_stack.apis.vector_io import QueryChunksResponse, VectorIO +from llama_stack.providers.datatypes import ToolsProtocolPrivate +from llama_stack.providers.utils.memory.vector_store import ( + content_from_doc, + make_overlapped_chunks, +) + +from .config import RagToolRuntimeConfig +from .context_retriever import generate_rag_query + +log = logging.getLogger(__name__) + + +def make_random_string(length: int = 8): + return "".join( + secrets.choice(string.ascii_letters + string.digits) for _ in range(length) + ) + + +class MemoryToolRuntimeImpl(ToolsProtocolPrivate, ToolRuntime, RAGToolRuntime): + def __init__( + self, + config: RagToolRuntimeConfig, + vector_io_api: VectorIO, + inference_api: Inference, + ): + self.config = config + self.vector_io_api = vector_io_api + self.inference_api = inference_api + + async def initialize(self): + pass + + async def shutdown(self): + pass + + async def insert( + self, + documents: List[RAGDocument], + vector_db_id: str, + chunk_size_in_tokens: int = 512, + ) -> None: + chunks = [] + for doc in documents: + content = await content_from_doc(doc) + chunks.extend( + make_overlapped_chunks( + doc.document_id, + content, + chunk_size_in_tokens, + chunk_size_in_tokens // 4, + ) + ) + + if not chunks: + return + + await self.vector_io_api.insert_chunks( + chunks=chunks, + vector_db_id=vector_db_id, + ) + + async def query( + self, + content: InterleavedContent, + vector_db_ids: List[str], + query_config: Optional[RAGQueryConfig] = None, + ) -> RAGQueryResult: + if not vector_db_ids: + return RAGQueryResult(content=None) + + query_config = query_config or RAGQueryConfig() + query = await generate_rag_query( + query_config.query_generator_config, + content, + inference_api=self.inference_api, + ) + tasks = [ + self.vector_io_api.query_chunks( + vector_db_id=vector_db_id, + query=query, + params={ + "max_chunks": query_config.max_chunks, + }, + ) + for vector_db_id in vector_db_ids + ] + results: List[QueryChunksResponse] = await asyncio.gather(*tasks) + chunks = [c for r in results for c in r.chunks] + scores = [s for r in results for s in r.scores] + + if not chunks: + return RAGQueryResult(content=None) + + # sort by score + chunks, scores = zip( + *sorted(zip(chunks, scores), key=lambda x: x[1], reverse=True) + ) + + tokens = 0 + picked = [] + for c in chunks[: query_config.max_chunks]: + metadata = c.metadata + tokens += metadata["token_count"] + if tokens > query_config.max_tokens_in_context: + log.error( + f"Using {len(picked)} chunks; reached max tokens in context: {tokens}", + ) + break + picked.append( + TextContentItem( + text=f"id:{metadata['document_id']}; content:{c.content}", + ) + ) + + return RAGQueryResult( + content=[ + TextContentItem( + text="Here are the retrieved documents for relevant context:\n=== START-RETRIEVED-CONTEXT ===\n", + ), + *picked, + TextContentItem( + text="\n=== END-RETRIEVED-CONTEXT ===\n", + ), + ], + ) + + async def list_runtime_tools( + self, tool_group_id: Optional[str] = None, mcp_endpoint: Optional[URL] = None + ) -> List[ToolDef]: + # Parameters are not listed since these methods are not yet invoked automatically + # by the LLM. The method is only implemented so things like /tools can list without + # encountering fatals. + return [ + ToolDef( + name="query_from_memory", + description="Retrieve context from memory", + ), + ToolDef( + name="insert_into_memory", + description="Insert documents into memory", + ), + ] + + async def invoke_tool( + self, tool_name: str, kwargs: Dict[str, Any] + ) -> ToolInvocationResult: + raise RuntimeError( + "This toolgroup should not be called generically but only through specific methods of the RAGToolRuntime protocol" + ) diff --git a/llama_stack/providers/remote/memory/__init__.py b/llama_stack/providers/inline/vector_io/__init__.py similarity index 100% rename from llama_stack/providers/remote/memory/__init__.py rename to llama_stack/providers/inline/vector_io/__init__.py diff --git a/llama_stack/providers/inline/memory/chroma/__init__.py b/llama_stack/providers/inline/vector_io/chroma/__init__.py similarity index 73% rename from llama_stack/providers/inline/memory/chroma/__init__.py rename to llama_stack/providers/inline/vector_io/chroma/__init__.py index 80620c780..68e28da63 100644 --- a/llama_stack/providers/inline/memory/chroma/__init__.py +++ b/llama_stack/providers/inline/vector_io/chroma/__init__.py @@ -14,8 +14,10 @@ from .config import ChromaInlineImplConfig async def get_provider_impl( config: ChromaInlineImplConfig, deps: Dict[Api, ProviderSpec] ): - from llama_stack.providers.remote.memory.chroma.chroma import ChromaMemoryAdapter + from llama_stack.providers.remote.vector_io.chroma.chroma import ( + ChromaVectorIOAdapter, + ) - impl = ChromaMemoryAdapter(config, deps[Api.inference]) + impl = ChromaVectorIOAdapter(config, deps[Api.inference]) await impl.initialize() return impl diff --git a/llama_stack/providers/inline/memory/chroma/config.py b/llama_stack/providers/inline/vector_io/chroma/config.py similarity index 100% rename from llama_stack/providers/inline/memory/chroma/config.py rename to llama_stack/providers/inline/vector_io/chroma/config.py diff --git a/llama_stack/providers/inline/memory/faiss/__init__.py b/llama_stack/providers/inline/vector_io/faiss/__init__.py similarity index 85% rename from llama_stack/providers/inline/memory/faiss/__init__.py rename to llama_stack/providers/inline/vector_io/faiss/__init__.py index 2d7ede3b1..32cf262fd 100644 --- a/llama_stack/providers/inline/memory/faiss/__init__.py +++ b/llama_stack/providers/inline/vector_io/faiss/__init__.py @@ -11,12 +11,12 @@ from .config import FaissImplConfig async def get_provider_impl(config: FaissImplConfig, deps: Dict[Api, ProviderSpec]): - from .faiss import FaissMemoryImpl + from .faiss import FaissVectorIOImpl assert isinstance( config, FaissImplConfig ), f"Unexpected config type: {type(config)}" - impl = FaissMemoryImpl(config, deps[Api.inference]) + impl = FaissVectorIOImpl(config, deps[Api.inference]) await impl.initialize() return impl diff --git a/llama_stack/providers/inline/memory/faiss/config.py b/llama_stack/providers/inline/vector_io/faiss/config.py similarity index 100% rename from llama_stack/providers/inline/memory/faiss/config.py rename to llama_stack/providers/inline/vector_io/faiss/config.py diff --git a/llama_stack/providers/inline/memory/faiss/faiss.py b/llama_stack/providers/inline/vector_io/faiss/faiss.py similarity index 62% rename from llama_stack/providers/inline/memory/faiss/faiss.py rename to llama_stack/providers/inline/vector_io/faiss/faiss.py index af398801a..db53302bb 100644 --- a/llama_stack/providers/inline/memory/faiss/faiss.py +++ b/llama_stack/providers/inline/vector_io/faiss/faiss.py @@ -17,35 +17,28 @@ import numpy as np from numpy.typing import NDArray from llama_stack.apis.inference import InterleavedContent -from llama_stack.apis.memory import ( - Chunk, - Memory, - MemoryBankDocument, - QueryDocumentsResponse, -) -from llama_stack.apis.memory_banks import MemoryBank, MemoryBankType, VectorMemoryBank -from llama_stack.providers.datatypes import Api, MemoryBanksProtocolPrivate +from llama_stack.apis.vector_dbs import VectorDB +from llama_stack.apis.vector_io import Chunk, QueryChunksResponse, VectorIO +from llama_stack.providers.datatypes import Api, VectorDBsProtocolPrivate from llama_stack.providers.utils.kvstore import kvstore_impl from llama_stack.providers.utils.memory.vector_store import ( - BankWithIndex, EmbeddingIndex, + VectorDBWithIndex, ) from .config import FaissImplConfig logger = logging.getLogger(__name__) -MEMORY_BANKS_PREFIX = "memory_banks:v2::" +VECTOR_DBS_PREFIX = "vector_dbs:v2::" FAISS_INDEX_PREFIX = "faiss_index:v2::" class FaissIndex(EmbeddingIndex): - id_by_index: Dict[int, str] chunk_by_index: Dict[int, str] def __init__(self, dimension: int, kvstore=None, bank_id: str = None): self.index = faiss.IndexFlatL2(dimension) - self.id_by_index = {} self.chunk_by_index = {} self.kvstore = kvstore self.bank_id = bank_id @@ -65,7 +58,6 @@ class FaissIndex(EmbeddingIndex): if stored_data: data = json.loads(stored_data) - self.id_by_index = {int(k): v for k, v in data["id_by_index"].items()} self.chunk_by_index = { int(k): Chunk.model_validate_json(v) for k, v in data["chunk_by_index"].items() @@ -82,7 +74,6 @@ class FaissIndex(EmbeddingIndex): buffer = io.BytesIO() np.savetxt(buffer, np_index) data = { - "id_by_index": self.id_by_index, "chunk_by_index": { k: v.model_dump_json() for k, v in self.chunk_by_index.items() }, @@ -108,10 +99,9 @@ class FaissIndex(EmbeddingIndex): f"Embedding dimension mismatch. Expected {self.index.d}, got {embedding_dim}" ) - indexlen = len(self.id_by_index) + indexlen = len(self.chunk_by_index) for i, chunk in enumerate(chunks): self.chunk_by_index[indexlen + i] = chunk - self.id_by_index[indexlen + i] = chunk.document_id self.index.add(np.array(embeddings).astype(np.float32)) @@ -120,7 +110,7 @@ class FaissIndex(EmbeddingIndex): async def query( self, embedding: NDArray, k: int, score_threshold: float - ) -> QueryDocumentsResponse: + ) -> QueryChunksResponse: distances, indices = self.index.search( embedding.reshape(1, -1).astype(np.float32), k ) @@ -133,10 +123,10 @@ class FaissIndex(EmbeddingIndex): chunks.append(self.chunk_by_index[int(i)]) scores.append(1.0 / float(d)) - return QueryDocumentsResponse(chunks=chunks, scores=scores) + return QueryChunksResponse(chunks=chunks, scores=scores) -class FaissMemoryImpl(Memory, MemoryBanksProtocolPrivate): +class FaissVectorIOImpl(VectorIO, VectorDBsProtocolPrivate): def __init__(self, config: FaissImplConfig, inference_api: Api.inference) -> None: self.config = config self.inference_api = inference_api @@ -146,77 +136,74 @@ class FaissMemoryImpl(Memory, MemoryBanksProtocolPrivate): async def initialize(self) -> None: self.kvstore = await kvstore_impl(self.config.kvstore) # Load existing banks from kvstore - start_key = MEMORY_BANKS_PREFIX - end_key = f"{MEMORY_BANKS_PREFIX}\xff" - stored_banks = await self.kvstore.range(start_key, end_key) + start_key = VECTOR_DBS_PREFIX + end_key = f"{VECTOR_DBS_PREFIX}\xff" + stored_vector_dbs = await self.kvstore.range(start_key, end_key) - for bank_data in stored_banks: - bank = VectorMemoryBank.model_validate_json(bank_data) - index = BankWithIndex( - bank, + for vector_db_data in stored_vector_dbs: + vector_db = VectorDB.model_validate_json(vector_db_data) + index = VectorDBWithIndex( + vector_db, await FaissIndex.create( - bank.embedding_dimension, self.kvstore, bank.identifier + vector_db.embedding_dimension, self.kvstore, vector_db.identifier ), self.inference_api, ) - self.cache[bank.identifier] = index + self.cache[vector_db.identifier] = index async def shutdown(self) -> None: # Cleanup if needed pass - async def register_memory_bank( + async def register_vector_db( self, - memory_bank: MemoryBank, + vector_db: VectorDB, ) -> None: - assert ( - memory_bank.memory_bank_type == MemoryBankType.vector.value - ), f"Only vector banks are supported {memory_bank.type}" - - # Store in kvstore - key = f"{MEMORY_BANKS_PREFIX}{memory_bank.identifier}" + key = f"{VECTOR_DBS_PREFIX}{vector_db.identifier}" await self.kvstore.set( key=key, - value=memory_bank.model_dump_json(), + value=vector_db.model_dump_json(), ) # Store in cache - self.cache[memory_bank.identifier] = BankWithIndex( - memory_bank, - await FaissIndex.create( - memory_bank.embedding_dimension, self.kvstore, memory_bank.identifier + self.cache[vector_db.identifier] = VectorDBWithIndex( + vector_db=vector_db, + index=await FaissIndex.create( + vector_db.embedding_dimension, self.kvstore, vector_db.identifier ), - self.inference_api, + inference_api=self.inference_api, ) - async def list_memory_banks(self) -> List[MemoryBank]: - return [i.bank for i in self.cache.values()] + async def list_vector_dbs(self) -> List[VectorDB]: + return [i.vector_db for i in self.cache.values()] - async def unregister_memory_bank(self, memory_bank_id: str) -> None: - await self.cache[memory_bank_id].index.delete() - del self.cache[memory_bank_id] - await self.kvstore.delete(f"{MEMORY_BANKS_PREFIX}{memory_bank_id}") + async def unregister_vector_db(self, vector_db_id: str) -> None: + await self.cache[vector_db_id].index.delete() + del self.cache[vector_db_id] + await self.kvstore.delete(f"{VECTOR_DBS_PREFIX}{vector_db_id}") - async def insert_documents( + async def insert_chunks( self, - bank_id: str, - documents: List[MemoryBankDocument], + vector_db_id: str, + chunks: List[Chunk], ttl_seconds: Optional[int] = None, ) -> None: - index = self.cache.get(bank_id) + index = self.cache.get(vector_db_id) if index is None: - raise ValueError(f"Bank {bank_id} not found. found: {self.cache.keys()}") + raise ValueError( + f"Vector DB {vector_db_id} not found. found: {self.cache.keys()}" + ) - await index.insert_documents(documents) + await index.insert_chunks(chunks) - async def query_documents( + async def query_chunks( self, - bank_id: str, + vector_db_id: str, query: InterleavedContent, params: Optional[Dict[str, Any]] = None, - ) -> QueryDocumentsResponse: - index = self.cache.get(bank_id) + ) -> QueryChunksResponse: + index = self.cache.get(vector_db_id) if index is None: - raise ValueError(f"Bank {bank_id} not found") + raise ValueError(f"Vector DB {vector_db_id} not found") - return await index.query_documents(query, params) + return await index.query_chunks(query, params) diff --git a/llama_stack/providers/registry/agents.py b/llama_stack/providers/registry/agents.py index 3e38b1adc..655303f98 100644 --- a/llama_stack/providers/registry/agents.py +++ b/llama_stack/providers/registry/agents.py @@ -33,8 +33,8 @@ def available_providers() -> List[ProviderSpec]: api_dependencies=[ Api.inference, Api.safety, - Api.memory, - Api.memory_banks, + Api.vector_io, + Api.vector_dbs, Api.tool_runtime, Api.tool_groups, ], diff --git a/llama_stack/providers/registry/post_training.py b/llama_stack/providers/registry/post_training.py index 3c5d06c05..3bcda6508 100644 --- a/llama_stack/providers/registry/post_training.py +++ b/llama_stack/providers/registry/post_training.py @@ -14,7 +14,7 @@ def available_providers() -> List[ProviderSpec]: InlineProviderSpec( api=Api.post_training, provider_type="inline::torchtune", - pip_packages=["torch", "torchtune", "torchao", "numpy"], + pip_packages=["torch", "torchtune==0.5.0", "torchao==0.8.0", "numpy"], module="llama_stack.providers.inline.post_training.torchtune", config_class="llama_stack.providers.inline.post_training.torchtune.TorchtunePostTrainingConfig", api_dependencies=[ diff --git a/llama_stack/providers/registry/telemetry.py b/llama_stack/providers/registry/telemetry.py index ba7e2f806..f3b41374c 100644 --- a/llama_stack/providers/registry/telemetry.py +++ b/llama_stack/providers/registry/telemetry.py @@ -24,7 +24,7 @@ def available_providers() -> List[ProviderSpec]: "opentelemetry-sdk", "opentelemetry-exporter-otlp-proto-http", ], - api_dependencies=[Api.datasetio], + optional_api_dependencies=[Api.datasetio], module="llama_stack.providers.inline.telemetry.meta_reference", config_class="llama_stack.providers.inline.telemetry.meta_reference.config.TelemetryConfig", ), diff --git a/llama_stack/providers/registry/tool_runtime.py b/llama_stack/providers/registry/tool_runtime.py index 40299edad..33d880f30 100644 --- a/llama_stack/providers/registry/tool_runtime.py +++ b/llama_stack/providers/registry/tool_runtime.py @@ -19,11 +19,11 @@ def available_providers() -> List[ProviderSpec]: return [ InlineProviderSpec( api=Api.tool_runtime, - provider_type="inline::memory-runtime", + provider_type="inline::rag-runtime", pip_packages=[], - module="llama_stack.providers.inline.tool_runtime.memory", - config_class="llama_stack.providers.inline.tool_runtime.memory.config.MemoryToolRuntimeConfig", - api_dependencies=[Api.memory, Api.memory_banks, Api.inference], + module="llama_stack.providers.inline.tool_runtime.rag", + config_class="llama_stack.providers.inline.tool_runtime.rag.config.RagToolRuntimeConfig", + api_dependencies=[Api.vector_io, Api.inference], ), InlineProviderSpec( api=Api.tool_runtime, diff --git a/llama_stack/providers/registry/memory.py b/llama_stack/providers/registry/vector_io.py similarity index 64% rename from llama_stack/providers/registry/memory.py rename to llama_stack/providers/registry/vector_io.py index 6867a9186..df7b7f4b3 100644 --- a/llama_stack/providers/registry/memory.py +++ b/llama_stack/providers/registry/vector_io.py @@ -38,78 +38,78 @@ EMBEDDING_DEPS = [ def available_providers() -> List[ProviderSpec]: return [ InlineProviderSpec( - api=Api.memory, + api=Api.vector_io, provider_type="inline::meta-reference", pip_packages=EMBEDDING_DEPS + ["faiss-cpu"], - module="llama_stack.providers.inline.memory.faiss", - config_class="llama_stack.providers.inline.memory.faiss.FaissImplConfig", + module="llama_stack.providers.inline.vector_io.faiss", + config_class="llama_stack.providers.inline.vector_io.faiss.FaissImplConfig", deprecation_warning="Please use the `inline::faiss` provider instead.", api_dependencies=[Api.inference], ), InlineProviderSpec( - api=Api.memory, + api=Api.vector_io, provider_type="inline::faiss", pip_packages=EMBEDDING_DEPS + ["faiss-cpu"], - module="llama_stack.providers.inline.memory.faiss", - config_class="llama_stack.providers.inline.memory.faiss.FaissImplConfig", + module="llama_stack.providers.inline.vector_io.faiss", + config_class="llama_stack.providers.inline.vector_io.faiss.FaissImplConfig", api_dependencies=[Api.inference], ), remote_provider_spec( - Api.memory, + Api.vector_io, AdapterSpec( adapter_type="chromadb", pip_packages=EMBEDDING_DEPS + ["chromadb-client"], - module="llama_stack.providers.remote.memory.chroma", - config_class="llama_stack.providers.remote.memory.chroma.ChromaRemoteImplConfig", + module="llama_stack.providers.remote.vector_io.chroma", + config_class="llama_stack.providers.remote.vector_io.chroma.ChromaRemoteImplConfig", ), api_dependencies=[Api.inference], ), InlineProviderSpec( - api=Api.memory, + api=Api.vector_io, provider_type="inline::chromadb", pip_packages=EMBEDDING_DEPS + ["chromadb"], - module="llama_stack.providers.inline.memory.chroma", - config_class="llama_stack.providers.inline.memory.chroma.ChromaInlineImplConfig", + module="llama_stack.providers.inline.vector_io.chroma", + config_class="llama_stack.providers.inline.vector_io.chroma.ChromaInlineImplConfig", api_dependencies=[Api.inference], ), remote_provider_spec( - Api.memory, + Api.vector_io, AdapterSpec( adapter_type="pgvector", pip_packages=EMBEDDING_DEPS + ["psycopg2-binary"], - module="llama_stack.providers.remote.memory.pgvector", - config_class="llama_stack.providers.remote.memory.pgvector.PGVectorConfig", + module="llama_stack.providers.remote.vector_io.pgvector", + config_class="llama_stack.providers.remote.vector_io.pgvector.PGVectorConfig", ), api_dependencies=[Api.inference], ), remote_provider_spec( - Api.memory, + Api.vector_io, AdapterSpec( adapter_type="weaviate", pip_packages=EMBEDDING_DEPS + ["weaviate-client"], - module="llama_stack.providers.remote.memory.weaviate", - config_class="llama_stack.providers.remote.memory.weaviate.WeaviateConfig", - provider_data_validator="llama_stack.providers.remote.memory.weaviate.WeaviateRequestProviderData", + module="llama_stack.providers.remote.vector_io.weaviate", + config_class="llama_stack.providers.remote.vector_io.weaviate.WeaviateConfig", + provider_data_validator="llama_stack.providers.remote.vector_io.weaviate.WeaviateRequestProviderData", ), api_dependencies=[Api.inference], ), remote_provider_spec( - api=Api.memory, + api=Api.vector_io, adapter=AdapterSpec( adapter_type="sample", pip_packages=[], - module="llama_stack.providers.remote.memory.sample", - config_class="llama_stack.providers.remote.memory.sample.SampleConfig", + module="llama_stack.providers.remote.vector_io.sample", + config_class="llama_stack.providers.remote.vector_io.sample.SampleConfig", ), api_dependencies=[], ), remote_provider_spec( - Api.memory, + Api.vector_io, AdapterSpec( adapter_type="qdrant", pip_packages=EMBEDDING_DEPS + ["qdrant-client"], - module="llama_stack.providers.remote.memory.qdrant", - config_class="llama_stack.providers.remote.memory.qdrant.QdrantConfig", + module="llama_stack.providers.remote.vector_io.qdrant", + config_class="llama_stack.providers.remote.vector_io.qdrant.QdrantConfig", ), api_dependencies=[Api.inference], ), diff --git a/llama_stack/providers/remote/inference/bedrock/bedrock.py b/llama_stack/providers/remote/inference/bedrock/bedrock.py index 59f30024e..10b51e86b 100644 --- a/llama_stack/providers/remote/inference/bedrock/bedrock.py +++ b/llama_stack/providers/remote/inference/bedrock/bedrock.py @@ -34,6 +34,7 @@ from llama_stack.providers.utils.inference.model_registry import ( ModelRegistryHelper, ) from llama_stack.providers.utils.inference.openai_compat import ( + get_sampling_strategy_options, OpenAICompatCompletionChoice, OpenAICompatCompletionResponse, process_chat_completion_response, @@ -166,16 +167,13 @@ class BedrockInferenceAdapter(ModelRegistryHelper, Inference): ) -> Dict: bedrock_model = request.model - inference_config = {} - param_mapping = { - "max_tokens": "max_gen_len", - "temperature": "temperature", - "top_p": "top_p", - } + sampling_params = request.sampling_params + options = get_sampling_strategy_options(sampling_params) - for k, v in param_mapping.items(): - if getattr(request.sampling_params, k): - inference_config[v] = getattr(request.sampling_params, k) + if sampling_params.max_tokens: + options["max_gen_len"] = sampling_params.max_tokens + if sampling_params.repetition_penalty > 0: + options["repetition_penalty"] = sampling_params.repetition_penalty prompt = await chat_completion_request_to_prompt( request, self.get_llama_model(request.model), self.formatter @@ -185,7 +183,7 @@ class BedrockInferenceAdapter(ModelRegistryHelper, Inference): "body": json.dumps( { "prompt": prompt, - **inference_config, + **options, } ), } diff --git a/llama_stack/providers/remote/inference/cerebras/cerebras.py b/llama_stack/providers/remote/inference/cerebras/cerebras.py index b78471787..0b6ce142c 100644 --- a/llama_stack/providers/remote/inference/cerebras/cerebras.py +++ b/llama_stack/providers/remote/inference/cerebras/cerebras.py @@ -9,6 +9,7 @@ from typing import AsyncGenerator, List, Optional, Union from cerebras.cloud.sdk import AsyncCerebras from llama_models.datatypes import CoreModelId from llama_models.llama3.api.chat_format import ChatFormat +from llama_models.llama3.api.datatypes import TopKSamplingStrategy from llama_models.llama3.api.tokenizer import Tokenizer from llama_stack.apis.common.content_types import InterleavedContent @@ -172,7 +173,9 @@ class CerebrasInferenceAdapter(ModelRegistryHelper, Inference): async def _get_params( self, request: Union[ChatCompletionRequest, CompletionRequest] ) -> dict: - if request.sampling_params and request.sampling_params.top_k: + if request.sampling_params and isinstance( + request.sampling_params.strategy, TopKSamplingStrategy + ): raise ValueError("`top_k` not supported by Cerebras") prompt = "" diff --git a/llama_stack/providers/remote/inference/fireworks/fireworks.py b/llama_stack/providers/remote/inference/fireworks/fireworks.py index b451f0264..5c98d2054 100644 --- a/llama_stack/providers/remote/inference/fireworks/fireworks.py +++ b/llama_stack/providers/remote/inference/fireworks/fireworks.py @@ -168,7 +168,10 @@ class FireworksInferenceAdapter( yield chunk def _build_options( - self, sampling_params: Optional[SamplingParams], fmt: ResponseFormat + self, + sampling_params: Optional[SamplingParams], + fmt: ResponseFormat, + logprobs: Optional[LogProbConfig], ) -> dict: options = get_sampling_options(sampling_params) options.setdefault("max_tokens", 512) @@ -187,6 +190,11 @@ class FireworksInferenceAdapter( else: raise ValueError(f"Unknown response format {fmt.type}") + if logprobs and logprobs.top_k: + options["logprobs"] = logprobs.top_k + if options["logprobs"] <= 0 or options["logprobs"] >= 5: + raise ValueError("Required range: 0 < top_k < 5") + return options async def chat_completion( @@ -257,7 +265,8 @@ class FireworksInferenceAdapter( if isinstance(request, ChatCompletionRequest): if media_present: input_dict["messages"] = [ - await convert_message_to_openai_dict(m) for m in request.messages + await convert_message_to_openai_dict(m, download=True) + for m in request.messages ] else: input_dict["prompt"] = await chat_completion_request_to_prompt( @@ -280,7 +289,9 @@ class FireworksInferenceAdapter( "model": request.model, **input_dict, "stream": request.stream, - **self._build_options(request.sampling_params, request.response_format), + **self._build_options( + request.sampling_params, request.response_format, request.logprobs + ), } async def embeddings( diff --git a/llama_stack/providers/remote/inference/groq/groq_utils.py b/llama_stack/providers/remote/inference/groq/groq_utils.py index 11f684847..bd1a07d7c 100644 --- a/llama_stack/providers/remote/inference/groq/groq_utils.py +++ b/llama_stack/providers/remote/inference/groq/groq_utils.py @@ -48,6 +48,9 @@ from llama_stack.apis.inference import ( ToolDefinition, ToolPromptFormat, ) +from llama_stack.providers.utils.inference.openai_compat import ( + get_sampling_strategy_options, +) def convert_chat_completion_request( @@ -77,6 +80,7 @@ def convert_chat_completion_request( if request.tool_prompt_format != ToolPromptFormat.json: warnings.warn("tool_prompt_format is not used by Groq. Ignoring.") + sampling_options = get_sampling_strategy_options(request.sampling_params) return CompletionCreateParams( model=request.model, messages=[_convert_message(message) for message in request.messages], @@ -84,8 +88,8 @@ def convert_chat_completion_request( frequency_penalty=None, stream=request.stream, max_tokens=request.sampling_params.max_tokens or None, - temperature=request.sampling_params.temperature, - top_p=request.sampling_params.top_p, + temperature=sampling_options.get("temperature", 1.0), + top_p=sampling_options.get("top_p", 1.0), tools=[_convert_groq_tool_definition(tool) for tool in request.tools or []], tool_choice=request.tool_choice.value if request.tool_choice else None, ) @@ -214,7 +218,7 @@ async def convert_chat_completion_response_stream( event=ChatCompletionResponseEvent( event_type=event_type, delta=ToolCallDelta( - content=tool_call, + tool_call=tool_call, parse_status=ToolCallParseStatus.succeeded, ), ) diff --git a/llama_stack/providers/remote/inference/nvidia/config.py b/llama_stack/providers/remote/inference/nvidia/config.py index 9e81211bd..d062e65d2 100644 --- a/llama_stack/providers/remote/inference/nvidia/config.py +++ b/llama_stack/providers/remote/inference/nvidia/config.py @@ -5,7 +5,7 @@ # the root directory of this source tree. import os -from typing import Optional +from typing import Any, Dict, Optional from llama_models.schema_utils import json_schema_type from pydantic import BaseModel, Field, SecretStr @@ -48,3 +48,10 @@ class NVIDIAConfig(BaseModel): default=60, description="Timeout for the HTTP requests", ) + + @classmethod + def sample_run_config(cls, **kwargs) -> Dict[str, Any]: + return { + "url": "https://integrate.api.nvidia.com", + "api_key": "${env.NVIDIA_API_KEY}", + } diff --git a/llama_stack/providers/remote/inference/nvidia/openai_utils.py b/llama_stack/providers/remote/inference/nvidia/openai_utils.py index 975812844..0f753f80d 100644 --- a/llama_stack/providers/remote/inference/nvidia/openai_utils.py +++ b/llama_stack/providers/remote/inference/nvidia/openai_utils.py @@ -8,6 +8,11 @@ import json import warnings from typing import Any, AsyncGenerator, Dict, Generator, List, Optional +from llama_models.datatypes import ( + GreedySamplingStrategy, + TopKSamplingStrategy, + TopPSamplingStrategy, +) from llama_models.llama3.api.datatypes import ( BuiltinTool, StopReason, @@ -263,19 +268,19 @@ def convert_chat_completion_request( if request.sampling_params.max_tokens: payload.update(max_tokens=request.sampling_params.max_tokens) - if request.sampling_params.strategy == "top_p": + strategy = request.sampling_params.strategy + if isinstance(strategy, TopPSamplingStrategy): nvext.update(top_k=-1) - payload.update(top_p=request.sampling_params.top_p) - elif request.sampling_params.strategy == "top_k": - if ( - request.sampling_params.top_k != -1 - and request.sampling_params.top_k < 1 - ): + payload.update(top_p=strategy.top_p) + payload.update(temperature=strategy.temperature) + elif isinstance(strategy, TopKSamplingStrategy): + if strategy.top_k != -1 and strategy.top_k < 1: warnings.warn("top_k must be -1 or >= 1") - nvext.update(top_k=request.sampling_params.top_k) - elif request.sampling_params.strategy == "greedy": + nvext.update(top_k=strategy.top_k) + elif isinstance(strategy, GreedySamplingStrategy): nvext.update(top_k=-1) - payload.update(temperature=request.sampling_params.temperature) + else: + raise ValueError(f"Unsupported sampling strategy: {strategy}") return payload @@ -500,7 +505,9 @@ async def convert_openai_chat_completion_stream( event=ChatCompletionResponseEvent( event_type=next(event_type), delta=ToolCallDelta( - content=_convert_openai_tool_calls(choice.delta.tool_calls)[0], + tool_call=_convert_openai_tool_calls(choice.delta.tool_calls)[ + 0 + ], parse_status=ToolCallParseStatus.succeeded, ), logprobs=_convert_openai_logprobs(choice.logprobs), diff --git a/llama_stack/providers/remote/inference/ollama/ollama.py b/llama_stack/providers/remote/inference/ollama/ollama.py index 38721ea22..6811d435b 100644 --- a/llama_stack/providers/remote/inference/ollama/ollama.py +++ b/llama_stack/providers/remote/inference/ollama/ollama.py @@ -172,6 +172,7 @@ class OllamaInferenceAdapter(Inference, ModelsProtocolPrivate): model=model.provider_resource_id, content=content, sampling_params=sampling_params, + response_format=response_format, stream=stream, logprobs=logprobs, ) diff --git a/llama_stack/providers/remote/inference/tgi/config.py b/llama_stack/providers/remote/inference/tgi/config.py index f05005b25..4f690dec6 100644 --- a/llama_stack/providers/remote/inference/tgi/config.py +++ b/llama_stack/providers/remote/inference/tgi/config.py @@ -15,10 +15,6 @@ class TGIImplConfig(BaseModel): url: str = Field( description="The URL for the TGI serving endpoint", ) - api_token: Optional[SecretStr] = Field( - default=None, - description="A bearer token if your TGI endpoint is protected.", - ) @classmethod def sample_run_config(cls, url: str = "${env.TGI_URL}", **kwargs): diff --git a/llama_stack/providers/remote/inference/tgi/tgi.py b/llama_stack/providers/remote/inference/tgi/tgi.py index 985fd3606..7f8c9d8ab 100644 --- a/llama_stack/providers/remote/inference/tgi/tgi.py +++ b/llama_stack/providers/remote/inference/tgi/tgi.py @@ -128,6 +128,12 @@ class _HfAdapter(Inference, ModelsProtocolPrivate): fmt: ResponseFormat = None, ): options = get_sampling_options(sampling_params) + # TGI does not support temperature=0 when using greedy sampling + # We set it to 1e-3 instead, anything lower outputs garbage from TGI + # We can use top_p sampling strategy to specify lower temperature + if abs(options["temperature"]) < 1e-10: + options["temperature"] = 1e-3 + # delete key "max_tokens" from options since its not supported by the API options.pop("max_tokens", None) if fmt: @@ -289,7 +295,7 @@ class TGIAdapter(_HfAdapter): async def initialize(self, config: TGIImplConfig) -> None: log.info(f"Initializing TGI client with url={config.url}") self.client = AsyncInferenceClient( - model=config.url, token=config.api_token.get_secret_value() + model=config.url, ) endpoint_info = await self.client.get_endpoint_info() self.max_tokens = endpoint_info["max_total_tokens"] diff --git a/llama_stack/providers/remote/inference/vllm/vllm.py b/llama_stack/providers/remote/inference/vllm/vllm.py index 317d05207..0cf16f013 100644 --- a/llama_stack/providers/remote/inference/vllm/vllm.py +++ b/llama_stack/providers/remote/inference/vllm/vllm.py @@ -41,6 +41,8 @@ from llama_stack.providers.utils.inference.openai_compat import ( get_sampling_options, process_chat_completion_response, process_chat_completion_stream_response, + process_completion_response, + process_completion_stream_response, ) from llama_stack.providers.utils.inference.prompt_adapter import ( chat_completion_request_to_prompt, @@ -92,7 +94,19 @@ class VLLMInferenceAdapter(Inference, ModelsProtocolPrivate): stream: Optional[bool] = False, logprobs: Optional[LogProbConfig] = None, ) -> Union[CompletionResponse, CompletionResponseStreamChunk]: - raise NotImplementedError("Completion not implemented for vLLM") + model = await self.model_store.get_model(model_id) + request = CompletionRequest( + model=model.provider_resource_id, + content=content, + sampling_params=sampling_params, + response_format=response_format, + stream=stream, + logprobs=logprobs, + ) + if stream: + return self._stream_completion(request) + else: + return await self._nonstream_completion(request) async def chat_completion( self, @@ -154,6 +168,26 @@ class VLLMInferenceAdapter(Inference, ModelsProtocolPrivate): ): yield chunk + async def _nonstream_completion( + self, request: CompletionRequest + ) -> CompletionResponse: + params = await self._get_params(request) + r = self.client.completions.create(**params) + return process_completion_response(r, self.formatter) + + async def _stream_completion(self, request: CompletionRequest) -> AsyncGenerator: + params = await self._get_params(request) + + # Wrapper for async generator similar + async def _to_async_generator(): + stream = self.client.completions.create(**params) + for chunk in stream: + yield chunk + + stream = _to_async_generator() + async for chunk in process_completion_stream_response(stream, self.formatter): + yield chunk + async def register_model(self, model: Model) -> Model: model = await self.register_helper.register_model(model) res = self.client.models.list() @@ -176,7 +210,6 @@ class VLLMInferenceAdapter(Inference, ModelsProtocolPrivate): media_present = request_has_media(request) if isinstance(request, ChatCompletionRequest): if media_present: - # vllm does not seem to work well with image urls, so we download the images input_dict["messages"] = [ await convert_message_to_openai_dict(m, download=True) for m in request.messages diff --git a/llama_stack/providers/remote/tool_runtime/bing_search/bing_search.py b/llama_stack/providers/remote/tool_runtime/bing_search/bing_search.py index 5114e06aa..677e29c12 100644 --- a/llama_stack/providers/remote/tool_runtime/bing_search/bing_search.py +++ b/llama_stack/providers/remote/tool_runtime/bing_search/bing_search.py @@ -68,7 +68,7 @@ class BingSearchToolRuntimeImpl( ] async def invoke_tool( - self, tool_name: str, args: Dict[str, Any] + self, tool_name: str, kwargs: Dict[str, Any] ) -> ToolInvocationResult: api_key = self._get_api_key() headers = { @@ -78,7 +78,7 @@ class BingSearchToolRuntimeImpl( "count": self.config.top_k, "textDecorations": True, "textFormat": "HTML", - "q": args["query"], + "q": kwargs["query"], } response = requests.get( diff --git a/llama_stack/providers/remote/tool_runtime/brave_search/brave_search.py b/llama_stack/providers/remote/tool_runtime/brave_search/brave_search.py index 016f746ea..1162cc900 100644 --- a/llama_stack/providers/remote/tool_runtime/brave_search/brave_search.py +++ b/llama_stack/providers/remote/tool_runtime/brave_search/brave_search.py @@ -68,7 +68,7 @@ class BraveSearchToolRuntimeImpl( ] async def invoke_tool( - self, tool_name: str, args: Dict[str, Any] + self, tool_name: str, kwargs: Dict[str, Any] ) -> ToolInvocationResult: api_key = self._get_api_key() url = "https://api.search.brave.com/res/v1/web/search" @@ -77,7 +77,7 @@ class BraveSearchToolRuntimeImpl( "Accept-Encoding": "gzip", "Accept": "application/json", } - payload = {"q": args["query"]} + payload = {"q": kwargs["query"]} response = requests.get(url=url, params=payload, headers=headers) response.raise_for_status() results = self._clean_brave_response(response.json()) diff --git a/llama_stack/providers/remote/tool_runtime/model_context_protocol/model_context_protocol.py b/llama_stack/providers/remote/tool_runtime/model_context_protocol/model_context_protocol.py index a304167e9..e0caec1d0 100644 --- a/llama_stack/providers/remote/tool_runtime/model_context_protocol/model_context_protocol.py +++ b/llama_stack/providers/remote/tool_runtime/model_context_protocol/model_context_protocol.py @@ -65,7 +65,7 @@ class ModelContextProtocolToolRuntimeImpl(ToolsProtocolPrivate, ToolRuntime): return tools async def invoke_tool( - self, tool_name: str, args: Dict[str, Any] + self, tool_name: str, kwargs: Dict[str, Any] ) -> ToolInvocationResult: tool = await self.tool_store.get_tool(tool_name) if tool.metadata is None or tool.metadata.get("endpoint") is None: @@ -77,7 +77,7 @@ class ModelContextProtocolToolRuntimeImpl(ToolsProtocolPrivate, ToolRuntime): async with sse_client(endpoint) as streams: async with ClientSession(*streams) as session: await session.initialize() - result = await session.call_tool(tool.identifier, args) + result = await session.call_tool(tool.identifier, kwargs) return ToolInvocationResult( content="\n".join([result.model_dump_json() for result in result.content]), diff --git a/llama_stack/providers/remote/tool_runtime/tavily_search/tavily_search.py b/llama_stack/providers/remote/tool_runtime/tavily_search/tavily_search.py index 82077193e..f5826c0ff 100644 --- a/llama_stack/providers/remote/tool_runtime/tavily_search/tavily_search.py +++ b/llama_stack/providers/remote/tool_runtime/tavily_search/tavily_search.py @@ -67,12 +67,12 @@ class TavilySearchToolRuntimeImpl( ] async def invoke_tool( - self, tool_name: str, args: Dict[str, Any] + self, tool_name: str, kwargs: Dict[str, Any] ) -> ToolInvocationResult: api_key = self._get_api_key() response = requests.post( "https://api.tavily.com/search", - json={"api_key": api_key, "query": args["query"]}, + json={"api_key": api_key, "query": kwargs["query"]}, ) return ToolInvocationResult( diff --git a/llama_stack/providers/remote/tool_runtime/wolfram_alpha/wolfram_alpha.py b/llama_stack/providers/remote/tool_runtime/wolfram_alpha/wolfram_alpha.py index 04ecfcc15..bf298c13e 100644 --- a/llama_stack/providers/remote/tool_runtime/wolfram_alpha/wolfram_alpha.py +++ b/llama_stack/providers/remote/tool_runtime/wolfram_alpha/wolfram_alpha.py @@ -68,11 +68,11 @@ class WolframAlphaToolRuntimeImpl( ] async def invoke_tool( - self, tool_name: str, args: Dict[str, Any] + self, tool_name: str, kwargs: Dict[str, Any] ) -> ToolInvocationResult: api_key = self._get_api_key() params = { - "input": args["query"], + "input": kwargs["query"], "appid": api_key, "format": "plaintext", "output": "json", diff --git a/llama_stack/providers/tests/memory/__init__.py b/llama_stack/providers/remote/vector_io/__init__.py similarity index 100% rename from llama_stack/providers/tests/memory/__init__.py rename to llama_stack/providers/remote/vector_io/__init__.py diff --git a/llama_stack/providers/remote/memory/chroma/__init__.py b/llama_stack/providers/remote/vector_io/chroma/__init__.py similarity index 81% rename from llama_stack/providers/remote/memory/chroma/__init__.py rename to llama_stack/providers/remote/vector_io/chroma/__init__.py index 581d60e75..d66a93ac7 100644 --- a/llama_stack/providers/remote/memory/chroma/__init__.py +++ b/llama_stack/providers/remote/vector_io/chroma/__init__.py @@ -14,8 +14,8 @@ from .config import ChromaRemoteImplConfig async def get_adapter_impl( config: ChromaRemoteImplConfig, deps: Dict[Api, ProviderSpec] ): - from .chroma import ChromaMemoryAdapter + from .chroma import ChromaVectorIOAdapter - impl = ChromaMemoryAdapter(config, deps[Api.inference]) + impl = ChromaVectorIOAdapter(config, deps[Api.inference]) await impl.initialize() return impl diff --git a/llama_stack/providers/remote/memory/chroma/chroma.py b/llama_stack/providers/remote/vector_io/chroma/chroma.py similarity index 62% rename from llama_stack/providers/remote/memory/chroma/chroma.py rename to llama_stack/providers/remote/vector_io/chroma/chroma.py index c04d775ca..724dc3f51 100644 --- a/llama_stack/providers/remote/memory/chroma/chroma.py +++ b/llama_stack/providers/remote/vector_io/chroma/chroma.py @@ -6,25 +6,20 @@ import asyncio import json import logging -from typing import List, Optional, Union +from typing import Any, Dict, List, Optional, Union from urllib.parse import urlparse import chromadb from numpy.typing import NDArray from llama_stack.apis.inference import InterleavedContent -from llama_stack.apis.memory import ( - Chunk, - Memory, - MemoryBankDocument, - QueryDocumentsResponse, -) -from llama_stack.apis.memory_banks import MemoryBank, MemoryBankType -from llama_stack.providers.datatypes import Api, MemoryBanksProtocolPrivate -from llama_stack.providers.inline.memory.chroma import ChromaInlineImplConfig +from llama_stack.apis.vector_dbs import VectorDB +from llama_stack.apis.vector_io import Chunk, QueryChunksResponse, VectorIO +from llama_stack.providers.datatypes import Api, VectorDBsProtocolPrivate +from llama_stack.providers.inline.vector_io.chroma import ChromaInlineImplConfig from llama_stack.providers.utils.memory.vector_store import ( - BankWithIndex, EmbeddingIndex, + VectorDBWithIndex, ) from .config import ChromaRemoteImplConfig @@ -61,7 +56,7 @@ class ChromaIndex(EmbeddingIndex): async def query( self, embedding: NDArray, k: int, score_threshold: float - ) -> QueryDocumentsResponse: + ) -> QueryChunksResponse: results = await maybe_await( self.collection.query( query_embeddings=[embedding.tolist()], @@ -85,19 +80,19 @@ class ChromaIndex(EmbeddingIndex): chunks.append(chunk) scores.append(1.0 / float(dist)) - return QueryDocumentsResponse(chunks=chunks, scores=scores) + return QueryChunksResponse(chunks=chunks, scores=scores) async def delete(self): await maybe_await(self.client.delete_collection(self.collection.name)) -class ChromaMemoryAdapter(Memory, MemoryBanksProtocolPrivate): +class ChromaVectorIOAdapter(VectorIO, VectorDBsProtocolPrivate): def __init__( self, config: Union[ChromaRemoteImplConfig, ChromaInlineImplConfig], inference_api: Api.inference, ) -> None: - log.info(f"Initializing ChromaMemoryAdapter with url: {config}") + log.info(f"Initializing ChromaVectorIOAdapter with url: {config}") self.config = config self.inference_api = inference_api @@ -123,60 +118,58 @@ class ChromaMemoryAdapter(Memory, MemoryBanksProtocolPrivate): async def shutdown(self) -> None: pass - async def register_memory_bank( + async def register_vector_db( self, - memory_bank: MemoryBank, + vector_db: VectorDB, ) -> None: - assert ( - memory_bank.memory_bank_type == MemoryBankType.vector.value - ), f"Only vector banks are supported {memory_bank.memory_bank_type}" - collection = await maybe_await( self.client.get_or_create_collection( - name=memory_bank.identifier, - metadata={"bank": memory_bank.model_dump_json()}, + name=vector_db.identifier, + metadata={"vector_db": vector_db.model_dump_json()}, ) ) - self.cache[memory_bank.identifier] = BankWithIndex( - memory_bank, ChromaIndex(self.client, collection), self.inference_api + self.cache[vector_db.identifier] = VectorDBWithIndex( + vector_db, ChromaIndex(self.client, collection), self.inference_api ) - async def unregister_memory_bank(self, memory_bank_id: str) -> None: - await self.cache[memory_bank_id].index.delete() - del self.cache[memory_bank_id] + async def unregister_vector_db(self, vector_db_id: str) -> None: + await self.cache[vector_db_id].index.delete() + del self.cache[vector_db_id] - async def insert_documents( + async def insert_chunks( self, - bank_id: str, - documents: List[MemoryBankDocument], - ttl_seconds: Optional[int] = None, + vector_db_id: str, + chunks: List[Chunk], + embeddings: NDArray, ) -> None: - index = await self._get_and_cache_bank_index(bank_id) + index = await self._get_and_cache_vector_db_index(vector_db_id) - await index.insert_documents(documents) + await index.insert_chunks(chunks, embeddings) - async def query_documents( + async def query_chunks( self, - bank_id: str, + vector_db_id: str, query: InterleavedContent, params: Optional[Dict[str, Any]] = None, - ) -> QueryDocumentsResponse: - index = await self._get_and_cache_bank_index(bank_id) + ) -> QueryChunksResponse: + index = await self._get_and_cache_vector_db_index(vector_db_id) - return await index.query_documents(query, params) + return await index.query_chunks(query, params) - async def _get_and_cache_bank_index(self, bank_id: str) -> BankWithIndex: - if bank_id in self.cache: - return self.cache[bank_id] + async def _get_and_cache_vector_db_index( + self, vector_db_id: str + ) -> VectorDBWithIndex: + if vector_db_id in self.cache: + return self.cache[vector_db_id] - bank = await self.memory_bank_store.get_memory_bank(bank_id) - if not bank: - raise ValueError(f"Bank {bank_id} not found in Llama Stack") - collection = await maybe_await(self.client.get_collection(bank_id)) + vector_db = await self.vector_db_store.get_vector_db(vector_db_id) + if not vector_db: + raise ValueError(f"Vector DB {vector_db_id} not found in Llama Stack") + collection = await maybe_await(self.client.get_collection(vector_db_id)) if not collection: - raise ValueError(f"Bank {bank_id} not found in Chroma") - index = BankWithIndex( - bank, ChromaIndex(self.client, collection), self.inference_api + raise ValueError(f"Vector DB {vector_db_id} not found in Chroma") + index = VectorDBWithIndex( + vector_db, ChromaIndex(self.client, collection), self.inference_api ) - self.cache[bank_id] = index + self.cache[vector_db_id] = index return index diff --git a/llama_stack/providers/remote/memory/chroma/config.py b/llama_stack/providers/remote/vector_io/chroma/config.py similarity index 100% rename from llama_stack/providers/remote/memory/chroma/config.py rename to llama_stack/providers/remote/vector_io/chroma/config.py diff --git a/llama_stack/providers/remote/memory/pgvector/__init__.py b/llama_stack/providers/remote/vector_io/pgvector/__init__.py similarity index 100% rename from llama_stack/providers/remote/memory/pgvector/__init__.py rename to llama_stack/providers/remote/vector_io/pgvector/__init__.py diff --git a/llama_stack/providers/remote/memory/pgvector/config.py b/llama_stack/providers/remote/vector_io/pgvector/config.py similarity index 100% rename from llama_stack/providers/remote/memory/pgvector/config.py rename to llama_stack/providers/remote/vector_io/pgvector/config.py diff --git a/llama_stack/providers/remote/memory/pgvector/pgvector.py b/llama_stack/providers/remote/vector_io/pgvector/pgvector.py similarity index 66% rename from llama_stack/providers/remote/memory/pgvector/pgvector.py rename to llama_stack/providers/remote/vector_io/pgvector/pgvector.py index b2c720b2c..3605f038c 100644 --- a/llama_stack/providers/remote/memory/pgvector/pgvector.py +++ b/llama_stack/providers/remote/vector_io/pgvector/pgvector.py @@ -12,21 +12,16 @@ from numpy.typing import NDArray from psycopg2 import sql from psycopg2.extras import execute_values, Json -from pydantic import BaseModel, parse_obj_as +from pydantic import BaseModel, TypeAdapter from llama_stack.apis.inference import InterleavedContent -from llama_stack.apis.memory import ( - Chunk, - Memory, - MemoryBankDocument, - QueryDocumentsResponse, -) -from llama_stack.apis.memory_banks import MemoryBank, MemoryBankType, VectorMemoryBank -from llama_stack.providers.datatypes import Api, MemoryBanksProtocolPrivate +from llama_stack.apis.vector_dbs import VectorDB +from llama_stack.apis.vector_io import Chunk, QueryChunksResponse, VectorIO +from llama_stack.providers.datatypes import Api, VectorDBsProtocolPrivate from llama_stack.providers.utils.memory.vector_store import ( - BankWithIndex, EmbeddingIndex, + VectorDBWithIndex, ) from .config import PGVectorConfig @@ -50,20 +45,20 @@ def upsert_models(cur, keys_models: List[Tuple[str, BaseModel]]): """ ) - values = [(key, Json(model.dict())) for key, model in keys_models] + values = [(key, Json(model.model_dump())) for key, model in keys_models] execute_values(cur, query, values, template="(%s, %s)") def load_models(cur, cls): cur.execute("SELECT key, data FROM metadata_store") rows = cur.fetchall() - return [parse_obj_as(cls, row["data"]) for row in rows] + return [TypeAdapter(cls).validate_python(row["data"]) for row in rows] class PGVectorIndex(EmbeddingIndex): - def __init__(self, bank: VectorMemoryBank, dimension: int, cursor): + def __init__(self, vector_db: VectorDB, dimension: int, cursor): self.cursor = cursor - self.table_name = f"vector_store_{bank.identifier}" + self.table_name = f"vector_store_{vector_db.identifier}" self.cursor.execute( f""" @@ -85,7 +80,7 @@ class PGVectorIndex(EmbeddingIndex): values.append( ( f"{chunk.document_id}:chunk-{i}", - Json(chunk.dict()), + Json(chunk.model_dump()), embeddings[i].tolist(), ) ) @@ -101,7 +96,7 @@ class PGVectorIndex(EmbeddingIndex): async def query( self, embedding: NDArray, k: int, score_threshold: float - ) -> QueryDocumentsResponse: + ) -> QueryChunksResponse: self.cursor.execute( f""" SELECT document, embedding <-> %s::vector AS distance @@ -119,13 +114,13 @@ class PGVectorIndex(EmbeddingIndex): chunks.append(Chunk(**doc)) scores.append(1.0 / float(dist)) - return QueryDocumentsResponse(chunks=chunks, scores=scores) + return QueryChunksResponse(chunks=chunks, scores=scores) async def delete(self): self.cursor.execute(f"DROP TABLE IF EXISTS {self.table_name}") -class PGVectorMemoryAdapter(Memory, MemoryBanksProtocolPrivate): +class PGVectorVectorDBAdapter(VectorIO, VectorDBsProtocolPrivate): def __init__(self, config: PGVectorConfig, inference_api: Api.inference) -> None: self.config = config self.inference_api = inference_api @@ -167,46 +162,45 @@ class PGVectorMemoryAdapter(Memory, MemoryBanksProtocolPrivate): async def shutdown(self) -> None: pass - async def register_memory_bank(self, memory_bank: MemoryBank) -> None: - assert ( - memory_bank.memory_bank_type == MemoryBankType.vector.value - ), f"Only vector banks are supported {memory_bank.memory_bank_type}" + async def register_vector_db(self, vector_db: VectorDB) -> None: + upsert_models(self.cursor, [(vector_db.identifier, vector_db)]) - upsert_models(self.cursor, [(memory_bank.identifier, memory_bank)]) - index = PGVectorIndex(memory_bank, memory_bank.embedding_dimension, self.cursor) - self.cache[memory_bank.identifier] = BankWithIndex( - memory_bank, index, self.inference_api + index = PGVectorIndex(vector_db, vector_db.embedding_dimension, self.cursor) + self.cache[vector_db.identifier] = VectorDBWithIndex( + vector_db, index, self.inference_api ) - async def unregister_memory_bank(self, memory_bank_id: str) -> None: - await self.cache[memory_bank_id].index.delete() - del self.cache[memory_bank_id] + async def unregister_vector_db(self, vector_db_id: str) -> None: + await self.cache[vector_db_id].index.delete() + del self.cache[vector_db_id] - async def insert_documents( + async def insert_chunks( self, - bank_id: str, - documents: List[MemoryBankDocument], + vector_db_id: str, + chunks: List[Chunk], ttl_seconds: Optional[int] = None, ) -> None: - index = await self._get_and_cache_bank_index(bank_id) - await index.insert_documents(documents) + index = await self._get_and_cache_vector_db_index(vector_db_id) + await index.insert_chunks(chunks) - async def query_documents( + async def query_chunks( self, - bank_id: str, + vector_db_id: str, query: InterleavedContent, params: Optional[Dict[str, Any]] = None, - ) -> QueryDocumentsResponse: - index = await self._get_and_cache_bank_index(bank_id) - return await index.query_documents(query, params) + ) -> QueryChunksResponse: + index = await self._get_and_cache_vector_db_index(vector_db_id) + return await index.query_chunks(query, params) - self.inference_api = inference_api + async def _get_and_cache_vector_db_index( + self, vector_db_id: str + ) -> VectorDBWithIndex: + if vector_db_id in self.cache: + return self.cache[vector_db_id] - async def _get_and_cache_bank_index(self, bank_id: str) -> BankWithIndex: - if bank_id in self.cache: - return self.cache[bank_id] - - bank = await self.memory_bank_store.get_memory_bank(bank_id) - index = PGVectorIndex(bank, bank.embedding_dimension, self.cursor) - self.cache[bank_id] = BankWithIndex(bank, index, self.inference_api) - return self.cache[bank_id] + vector_db = await self.vector_db_store.get_vector_db(vector_db_id) + index = PGVectorIndex(vector_db, vector_db.embedding_dimension, self.cursor) + self.cache[vector_db_id] = VectorDBWithIndex( + vector_db, index, self.inference_api + ) + return self.cache[vector_db_id] diff --git a/llama_stack/providers/remote/memory/qdrant/__init__.py b/llama_stack/providers/remote/vector_io/qdrant/__init__.py similarity index 100% rename from llama_stack/providers/remote/memory/qdrant/__init__.py rename to llama_stack/providers/remote/vector_io/qdrant/__init__.py diff --git a/llama_stack/providers/remote/memory/qdrant/config.py b/llama_stack/providers/remote/vector_io/qdrant/config.py similarity index 100% rename from llama_stack/providers/remote/memory/qdrant/config.py rename to llama_stack/providers/remote/vector_io/qdrant/config.py diff --git a/llama_stack/providers/remote/memory/qdrant/qdrant.py b/llama_stack/providers/remote/vector_io/qdrant/qdrant.py similarity index 67% rename from llama_stack/providers/remote/memory/qdrant/qdrant.py rename to llama_stack/providers/remote/vector_io/qdrant/qdrant.py index b1d5bd7fa..d3257b4c9 100644 --- a/llama_stack/providers/remote/memory/qdrant/qdrant.py +++ b/llama_stack/providers/remote/vector_io/qdrant/qdrant.py @@ -13,19 +13,14 @@ from qdrant_client import AsyncQdrantClient, models from qdrant_client.models import PointStruct from llama_stack.apis.inference import InterleavedContent -from llama_stack.apis.memory import ( - Chunk, - Memory, - MemoryBankDocument, - QueryDocumentsResponse, -) -from llama_stack.apis.memory_banks import MemoryBank, MemoryBankType -from llama_stack.providers.datatypes import Api, MemoryBanksProtocolPrivate -from llama_stack.providers.remote.memory.qdrant.config import QdrantConfig +from llama_stack.apis.vector_dbs import VectorDB +from llama_stack.apis.vector_io import Chunk, QueryChunksResponse, VectorIO +from llama_stack.providers.datatypes import Api, VectorDBsProtocolPrivate from llama_stack.providers.utils.memory.vector_store import ( - BankWithIndex, EmbeddingIndex, + VectorDBWithIndex, ) +from .config import QdrantConfig log = logging.getLogger(__name__) CHUNK_ID_KEY = "_chunk_id" @@ -76,7 +71,7 @@ class QdrantIndex(EmbeddingIndex): async def query( self, embedding: NDArray, k: int, score_threshold: float - ) -> QueryDocumentsResponse: + ) -> QueryChunksResponse: results = ( await self.client.query_points( collection_name=self.collection_name, @@ -101,10 +96,10 @@ class QdrantIndex(EmbeddingIndex): chunks.append(chunk) scores.append(point.score) - return QueryDocumentsResponse(chunks=chunks, scores=scores) + return QueryChunksResponse(chunks=chunks, scores=scores) -class QdrantVectorMemoryAdapter(Memory, MemoryBanksProtocolPrivate): +class QdrantVectorDBAdapter(VectorIO, VectorDBsProtocolPrivate): def __init__(self, config: QdrantConfig, inference_api: Api.inference) -> None: self.config = config self.client = AsyncQdrantClient(**self.config.model_dump(exclude_none=True)) @@ -117,58 +112,56 @@ class QdrantVectorMemoryAdapter(Memory, MemoryBanksProtocolPrivate): async def shutdown(self) -> None: self.client.close() - async def register_memory_bank( + async def register_vector_db( self, - memory_bank: MemoryBank, + vector_db: VectorDB, ) -> None: - assert ( - memory_bank.memory_bank_type == MemoryBankType.vector - ), f"Only vector banks are supported {memory_bank.memory_bank_type}" - - index = BankWithIndex( - bank=memory_bank, - index=QdrantIndex(self.client, memory_bank.identifier), + index = VectorDBWithIndex( + vector_db=vector_db, + index=QdrantIndex(self.client, vector_db.identifier), inference_api=self.inference_api, ) - self.cache[memory_bank.identifier] = index + self.cache[vector_db.identifier] = index - async def _get_and_cache_bank_index(self, bank_id: str) -> Optional[BankWithIndex]: - if bank_id in self.cache: - return self.cache[bank_id] + async def _get_and_cache_vector_db_index( + self, vector_db_id: str + ) -> Optional[VectorDBWithIndex]: + if vector_db_id in self.cache: + return self.cache[vector_db_id] - bank = await self.memory_bank_store.get_memory_bank(bank_id) - if not bank: - raise ValueError(f"Bank {bank_id} not found") + vector_db = await self.vector_db_store.get_vector_db(vector_db_id) + if not vector_db: + raise ValueError(f"Vector DB {vector_db_id} not found") - index = BankWithIndex( - bank=bank, - index=QdrantIndex(client=self.client, collection_name=bank_id), + index = VectorDBWithIndex( + vector_db=vector_db, + index=QdrantIndex(client=self.client, collection_name=vector_db.identifier), inference_api=self.inference_api, ) - self.cache[bank_id] = index + self.cache[vector_db_id] = index return index - async def insert_documents( + async def insert_chunks( self, - bank_id: str, - documents: List[MemoryBankDocument], + vector_db_id: str, + chunks: List[Chunk], ttl_seconds: Optional[int] = None, ) -> None: - index = await self._get_and_cache_bank_index(bank_id) + index = await self._get_and_cache_vector_db_index(vector_db_id) if not index: - raise ValueError(f"Bank {bank_id} not found") + raise ValueError(f"Vector DB {vector_db_id} not found") - await index.insert_documents(documents) + await index.insert_chunks(chunks) - async def query_documents( + async def query_chunks( self, - bank_id: str, + vector_db_id: str, query: InterleavedContent, params: Optional[Dict[str, Any]] = None, - ) -> QueryDocumentsResponse: - index = await self._get_and_cache_bank_index(bank_id) + ) -> QueryChunksResponse: + index = await self._get_and_cache_vector_db_index(vector_db_id) if not index: - raise ValueError(f"Bank {bank_id} not found") + raise ValueError(f"Vector DB {vector_db_id} not found") - return await index.query_documents(query, params) + return await index.query_chunks(query, params) diff --git a/llama_stack/providers/remote/memory/sample/__init__.py b/llama_stack/providers/remote/vector_io/sample/__init__.py similarity index 100% rename from llama_stack/providers/remote/memory/sample/__init__.py rename to llama_stack/providers/remote/vector_io/sample/__init__.py diff --git a/llama_stack/providers/remote/memory/sample/config.py b/llama_stack/providers/remote/vector_io/sample/config.py similarity index 100% rename from llama_stack/providers/remote/memory/sample/config.py rename to llama_stack/providers/remote/vector_io/sample/config.py diff --git a/llama_stack/providers/remote/memory/sample/sample.py b/llama_stack/providers/remote/vector_io/sample/sample.py similarity index 55% rename from llama_stack/providers/remote/memory/sample/sample.py rename to llama_stack/providers/remote/vector_io/sample/sample.py index b051eb544..e311be39d 100644 --- a/llama_stack/providers/remote/memory/sample/sample.py +++ b/llama_stack/providers/remote/vector_io/sample/sample.py @@ -4,19 +4,22 @@ # This source code is licensed under the terms described in the LICENSE file in # the root directory of this source tree. -from llama_stack.apis.memory import Memory -from llama_stack.apis.memory_banks import MemoryBank +from llama_stack.apis.vector_dbs import VectorDB +from llama_stack.apis.vector_io import VectorIO from .config import SampleConfig -class SampleMemoryImpl(Memory): +class SampleMemoryImpl(VectorIO): def __init__(self, config: SampleConfig): self.config = config - async def register_memory_bank(self, memory_bank: MemoryBank) -> None: - # these are the memory banks the Llama Stack will use to route requests to this provider + async def register_vector_db(self, vector_db: VectorDB) -> None: + # these are the vector dbs the Llama Stack will use to route requests to this provider # perform validation here if necessary pass async def initialize(self): pass + + async def shutdown(self): + pass diff --git a/llama_stack/providers/remote/memory/weaviate/__init__.py b/llama_stack/providers/remote/vector_io/weaviate/__init__.py similarity index 100% rename from llama_stack/providers/remote/memory/weaviate/__init__.py rename to llama_stack/providers/remote/vector_io/weaviate/__init__.py diff --git a/llama_stack/providers/remote/memory/weaviate/config.py b/llama_stack/providers/remote/vector_io/weaviate/config.py similarity index 100% rename from llama_stack/providers/remote/memory/weaviate/config.py rename to llama_stack/providers/remote/vector_io/weaviate/config.py diff --git a/llama_stack/providers/remote/memory/weaviate/weaviate.py b/llama_stack/providers/remote/vector_io/weaviate/weaviate.py similarity index 68% rename from llama_stack/providers/remote/memory/weaviate/weaviate.py rename to llama_stack/providers/remote/vector_io/weaviate/weaviate.py index f1433090d..ea9ce5185 100644 --- a/llama_stack/providers/remote/memory/weaviate/weaviate.py +++ b/llama_stack/providers/remote/vector_io/weaviate/weaviate.py @@ -15,18 +15,13 @@ from weaviate.classes.init import Auth from weaviate.classes.query import Filter from llama_stack.apis.common.content_types import InterleavedContent -from llama_stack.apis.memory import ( - Chunk, - Memory, - MemoryBankDocument, - QueryDocumentsResponse, -) -from llama_stack.apis.memory_banks import MemoryBank, MemoryBankType +from llama_stack.apis.vector_dbs import VectorDB +from llama_stack.apis.vector_io import Chunk, QueryChunksResponse, VectorIO from llama_stack.distribution.request_headers import NeedsRequestProviderData -from llama_stack.providers.datatypes import Api, MemoryBanksProtocolPrivate +from llama_stack.providers.datatypes import Api, VectorDBsProtocolPrivate from llama_stack.providers.utils.memory.vector_store import ( - BankWithIndex, EmbeddingIndex, + VectorDBWithIndex, ) from .config import WeaviateConfig, WeaviateRequestProviderData @@ -49,7 +44,7 @@ class WeaviateIndex(EmbeddingIndex): data_objects.append( wvc.data.DataObject( properties={ - "chunk_content": chunk.json(), + "chunk_content": chunk.model_dump_json(), }, vector=embeddings[i].tolist(), ) @@ -63,7 +58,7 @@ class WeaviateIndex(EmbeddingIndex): async def query( self, embedding: NDArray, k: int, score_threshold: float - ) -> QueryDocumentsResponse: + ) -> QueryChunksResponse: collection = self.client.collections.get(self.collection_name) results = collection.query.near_vector( @@ -86,7 +81,7 @@ class WeaviateIndex(EmbeddingIndex): chunks.append(chunk) scores.append(1.0 / doc.metadata.distance) - return QueryDocumentsResponse(chunks=chunks, scores=scores) + return QueryChunksResponse(chunks=chunks, scores=scores) async def delete(self, chunk_ids: List[str]) -> None: collection = self.client.collections.get(self.collection_name) @@ -96,9 +91,9 @@ class WeaviateIndex(EmbeddingIndex): class WeaviateMemoryAdapter( - Memory, + VectorIO, NeedsRequestProviderData, - MemoryBanksProtocolPrivate, + VectorDBsProtocolPrivate, ): def __init__(self, config: WeaviateConfig, inference_api: Api.inference) -> None: self.config = config @@ -129,20 +124,16 @@ class WeaviateMemoryAdapter( for client in self.client_cache.values(): client.close() - async def register_memory_bank( + async def register_vector_db( self, - memory_bank: MemoryBank, + vector_db: VectorDB, ) -> None: - assert ( - memory_bank.memory_bank_type == MemoryBankType.vector.value - ), f"Only vector banks are supported {memory_bank.memory_bank_type}" - client = self._get_client() # Create collection if it doesn't exist - if not client.collections.exists(memory_bank.identifier): + if not client.collections.exists(vector_db.identifier): client.collections.create( - name=memory_bank.identifier, + name=vector_db.identifier, vectorizer_config=wvc.config.Configure.Vectorizer.none(), properties=[ wvc.config.Property( @@ -152,52 +143,54 @@ class WeaviateMemoryAdapter( ], ) - self.cache[memory_bank.identifier] = BankWithIndex( - memory_bank, - WeaviateIndex(client=client, collection_name=memory_bank.identifier), + self.cache[vector_db.identifier] = VectorDBWithIndex( + vector_db, + WeaviateIndex(client=client, collection_name=vector_db.identifier), self.inference_api, ) - async def _get_and_cache_bank_index(self, bank_id: str) -> Optional[BankWithIndex]: - if bank_id in self.cache: - return self.cache[bank_id] + async def _get_and_cache_vector_db_index( + self, vector_db_id: str + ) -> Optional[VectorDBWithIndex]: + if vector_db_id in self.cache: + return self.cache[vector_db_id] - bank = await self.memory_bank_store.get_memory_bank(bank_id) - if not bank: - raise ValueError(f"Bank {bank_id} not found") + vector_db = await self.vector_db_store.get_vector_db(vector_db_id) + if not vector_db: + raise ValueError(f"Vector DB {vector_db_id} not found") client = self._get_client() - if not client.collections.exists(bank.identifier): - raise ValueError(f"Collection with name `{bank.identifier}` not found") + if not client.collections.exists(vector_db.identifier): + raise ValueError(f"Collection with name `{vector_db.identifier}` not found") - index = BankWithIndex( - bank=bank, - index=WeaviateIndex(client=client, collection_name=bank_id), + index = VectorDBWithIndex( + vector_db=vector_db, + index=WeaviateIndex(client=client, collection_name=vector_db.identifier), inference_api=self.inference_api, ) - self.cache[bank_id] = index + self.cache[vector_db_id] = index return index - async def insert_documents( + async def insert_chunks( self, - bank_id: str, - documents: List[MemoryBankDocument], + vector_db_id: str, + chunks: List[Chunk], ttl_seconds: Optional[int] = None, ) -> None: - index = await self._get_and_cache_bank_index(bank_id) + index = await self._get_and_cache_vector_db_index(vector_db_id) if not index: - raise ValueError(f"Bank {bank_id} not found") + raise ValueError(f"Vector DB {vector_db_id} not found") - await index.insert_documents(documents) + await index.insert_chunks(chunks) - async def query_documents( + async def query_chunks( self, - bank_id: str, + vector_db_id: str, query: InterleavedContent, params: Optional[Dict[str, Any]] = None, - ) -> QueryDocumentsResponse: - index = await self._get_and_cache_bank_index(bank_id) + ) -> QueryChunksResponse: + index = await self._get_and_cache_vector_db_index(vector_db_id) if not index: - raise ValueError(f"Bank {bank_id} not found") + raise ValueError(f"Vector DB {vector_db_id} not found") - return await index.query_documents(query, params) + return await index.query_chunks(query, params) diff --git a/llama_stack/providers/tests/README.md b/llama_stack/providers/tests/README.md index 4b406b321..e4e94a3fd 100644 --- a/llama_stack/providers/tests/README.md +++ b/llama_stack/providers/tests/README.md @@ -10,6 +10,8 @@ We use `pytest` and all of its dynamism to enable the features needed. Specifica - We use `pytest_configure` to make sure we dynamically add appropriate marks based on the fixtures we make. +- We use `pytest_collection_modifyitems` to filter tests based on the test config (if specified). + ## Common options All tests support a `--providers` option which can be a string of the form `api1=provider_fixture1,api2=provider_fixture2`. So, when testing safety (which need inference and safety APIs) you can use `--providers inference=together,safety=meta_reference` to use these fixtures in concert. @@ -73,3 +75,15 @@ If you wanted to test a remotely hosted stack, you can use `-m remote` as follow pytest -s -m remote llama_stack/providers/tests/agents/test_agents.py \ --env REMOTE_STACK_URL=<...> ``` + +## Test Config +If you want to run a test suite with a custom set of tests and parametrizations, you can define a YAML test config under llama_stack/providers/tests/ folder and pass the filename through `--config` option as follows: + +``` +pytest llama_stack/providers/tests/ --config=ci_test_config.yaml +``` + +### Test config format +Currently, we support test config on inference, agents and memory api tests. + +Example format of test config can be found in ci_test_config.yaml. diff --git a/llama_stack/providers/tests/agents/conftest.py b/llama_stack/providers/tests/agents/conftest.py index ecd05dcf8..9c115e3a1 100644 --- a/llama_stack/providers/tests/agents/conftest.py +++ b/llama_stack/providers/tests/agents/conftest.py @@ -6,11 +6,16 @@ import pytest -from ..conftest import get_provider_fixture_overrides +from ..conftest import ( + get_provider_fixture_overrides, + get_provider_fixture_overrides_from_test_config, + get_test_config_for_api, +) from ..inference.fixtures import INFERENCE_FIXTURES -from ..memory.fixtures import MEMORY_FIXTURES from ..safety.fixtures import SAFETY_FIXTURES, safety_model_from_shield + from ..tools.fixtures import TOOL_RUNTIME_FIXTURES +from ..vector_io.fixtures import VECTOR_IO_FIXTURES from .fixtures import AGENTS_FIXTURES DEFAULT_PROVIDER_COMBINATIONS = [ @@ -18,7 +23,7 @@ DEFAULT_PROVIDER_COMBINATIONS = [ { "inference": "meta_reference", "safety": "llama_guard", - "memory": "faiss", + "vector_io": "faiss", "agents": "meta_reference", "tool_runtime": "memory_and_search", }, @@ -29,7 +34,7 @@ DEFAULT_PROVIDER_COMBINATIONS = [ { "inference": "ollama", "safety": "llama_guard", - "memory": "faiss", + "vector_io": "faiss", "agents": "meta_reference", "tool_runtime": "memory_and_search", }, @@ -41,7 +46,7 @@ DEFAULT_PROVIDER_COMBINATIONS = [ "inference": "together", "safety": "llama_guard", # make this work with Weaviate which is what the together distro supports - "memory": "faiss", + "vector_io": "faiss", "agents": "meta_reference", "tool_runtime": "memory_and_search", }, @@ -52,7 +57,7 @@ DEFAULT_PROVIDER_COMBINATIONS = [ { "inference": "fireworks", "safety": "llama_guard", - "memory": "faiss", + "vector_io": "faiss", "agents": "meta_reference", "tool_runtime": "memory_and_search", }, @@ -63,7 +68,7 @@ DEFAULT_PROVIDER_COMBINATIONS = [ { "inference": "remote", "safety": "remote", - "memory": "remote", + "vector_io": "remote", "agents": "remote", "tool_runtime": "memory_and_search", }, @@ -81,23 +86,15 @@ def pytest_configure(config): ) -def pytest_addoption(parser): - parser.addoption( - "--inference-model", - action="store", - default="meta-llama/Llama-3.2-3B-Instruct", - help="Specify the inference model to use for testing", - ) - parser.addoption( - "--safety-shield", - action="store", - default="meta-llama/Llama-Guard-3-1B", - help="Specify the safety shield to use for testing", - ) - - def pytest_generate_tests(metafunc): - shield_id = metafunc.config.getoption("--safety-shield") + test_config = get_test_config_for_api(metafunc.config, "agents") + shield_id = getattr( + test_config, "safety_shield", None + ) or metafunc.config.getoption("--safety-shield") + inference_models = getattr(test_config, "inference_models", None) or [ + metafunc.config.getoption("--inference-model") + ] + if "safety_shield" in metafunc.fixturenames: metafunc.parametrize( "safety_shield", @@ -105,8 +102,7 @@ def pytest_generate_tests(metafunc): indirect=True, ) if "inference_model" in metafunc.fixturenames: - inference_model = metafunc.config.getoption("--inference-model") - models = set({inference_model}) + models = set(inference_models) if safety_model := safety_model_from_shield(shield_id): models.add(safety_model) @@ -119,12 +115,15 @@ def pytest_generate_tests(metafunc): available_fixtures = { "inference": INFERENCE_FIXTURES, "safety": SAFETY_FIXTURES, - "memory": MEMORY_FIXTURES, + "vector_io": VECTOR_IO_FIXTURES, "agents": AGENTS_FIXTURES, "tool_runtime": TOOL_RUNTIME_FIXTURES, } combinations = ( - get_provider_fixture_overrides(metafunc.config, available_fixtures) + get_provider_fixture_overrides_from_test_config( + metafunc.config, "agents", DEFAULT_PROVIDER_COMBINATIONS + ) + or get_provider_fixture_overrides(metafunc.config, available_fixtures) or DEFAULT_PROVIDER_COMBINATIONS ) metafunc.parametrize("agents_stack", combinations, indirect=True) diff --git a/llama_stack/providers/tests/agents/fixtures.py b/llama_stack/providers/tests/agents/fixtures.py index 1b1781f36..bb4a6e6a3 100644 --- a/llama_stack/providers/tests/agents/fixtures.py +++ b/llama_stack/providers/tests/agents/fixtures.py @@ -69,7 +69,7 @@ async def agents_stack( providers = {} provider_data = {} - for key in ["inference", "safety", "memory", "agents", "tool_runtime"]: + for key in ["inference", "safety", "vector_io", "agents", "tool_runtime"]: fixture = request.getfixturevalue(f"{key}_{fixture_dict[key]}") providers[key] = fixture.providers if key == "inference": @@ -118,7 +118,7 @@ async def agents_stack( ) test_stack = await construct_stack_for_test( - [Api.agents, Api.inference, Api.safety, Api.memory, Api.tool_runtime], + [Api.agents, Api.inference, Api.safety, Api.vector_io, Api.tool_runtime], providers, provider_data, models=models, diff --git a/llama_stack/providers/tests/agents/test_agents.py b/llama_stack/providers/tests/agents/test_agents.py index 27fb90572..68ee9133c 100644 --- a/llama_stack/providers/tests/agents/test_agents.py +++ b/llama_stack/providers/tests/agents/test_agents.py @@ -7,6 +7,7 @@ import os import pytest +from llama_models.datatypes import SamplingParams, TopPSamplingStrategy from llama_models.llama3.api.datatypes import BuiltinTool from llama_stack.apis.agents import ( @@ -22,7 +23,8 @@ from llama_stack.apis.agents import ( ToolExecutionStep, Turn, ) -from llama_stack.apis.inference import CompletionMessage, SamplingParams, UserMessage + +from llama_stack.apis.inference import CompletionMessage, UserMessage from llama_stack.apis.safety import ViolationLevel from llama_stack.providers.datatypes import Api @@ -42,7 +44,9 @@ def common_params(inference_model): model=inference_model, instructions="You are a helpful assistant.", enable_session_persistence=True, - sampling_params=SamplingParams(temperature=0.7, top_p=0.95), + sampling_params=SamplingParams( + strategy=TopPSamplingStrategy(temperature=0.7, top_p=0.95) + ), input_shields=[], output_shields=[], toolgroups=[], @@ -180,7 +184,7 @@ class TestAgents: agent_config = AgentConfig( **{ **common_params, - "toolgroups": ["builtin::memory"], + "toolgroups": ["builtin::rag"], "tool_choice": ToolChoice.auto, } ) @@ -210,9 +214,11 @@ class TestAgents: turn_response = [ chunk async for chunk in await agents_impl.create_agent_turn(**turn_request) ] - assert len(turn_response) > 0 + # FIXME: we need to check the content of the turn response and ensure + # RAG actually worked + @pytest.mark.asyncio async def test_create_agent_turn_with_tavily_search( self, agents_stack, search_query_messages, common_params diff --git a/llama_stack/providers/tests/agents/test_persistence.py b/llama_stack/providers/tests/agents/test_persistence.py index 38eb7de55..e6b1470ef 100644 --- a/llama_stack/providers/tests/agents/test_persistence.py +++ b/llama_stack/providers/tests/agents/test_persistence.py @@ -9,7 +9,9 @@ import pytest from llama_stack.apis.agents import AgentConfig, Turn from llama_stack.apis.inference import SamplingParams, UserMessage from llama_stack.providers.datatypes import Api -from llama_stack.providers.utils.kvstore import kvstore_impl, SqliteKVStoreConfig +from llama_stack.providers.utils.kvstore import kvstore_impl +from llama_stack.providers.utils.kvstore.config import SqliteKVStoreConfig + from .fixtures import pick_inference_model from .utils import create_agent_session diff --git a/llama_stack/providers/tests/ci_test_config.yaml b/llama_stack/providers/tests/ci_test_config.yaml new file mode 100644 index 000000000..3edcd38bf --- /dev/null +++ b/llama_stack/providers/tests/ci_test_config.yaml @@ -0,0 +1,55 @@ +inference: + tests: + - inference/test_vision_inference.py::test_vision_chat_completion_streaming + - inference/test_vision_inference.py::test_vision_chat_completion_non_streaming + - inference/test_text_inference.py::test_structured_output + - inference/test_text_inference.py::test_chat_completion_streaming + - inference/test_text_inference.py::test_chat_completion_non_streaming + - inference/test_text_inference.py::test_chat_completion_with_tool_calling + - inference/test_text_inference.py::test_chat_completion_with_tool_calling_streaming + + scenarios: + - provider_fixtures: + inference: ollama + - fixture_combo_id: fireworks + - provider_fixtures: + inference: together + # - inference: tgi + # - inference: vllm_remote + + inference_models: + - meta-llama/Llama-3.1-8B-Instruct + - meta-llama/Llama-3.2-11B-Vision-Instruct + + +agents: + tests: + - agents/test_agents.py::test_agent_turns_with_safety + - agents/test_agents.py::test_rag_agent + + scenarios: + - fixture_combo_id: ollama + - fixture_combo_id: together + - fixture_combo_id: fireworks + + inference_models: + - meta-llama/Llama-3.2-1B-Instruct + + safety_shield: meta-llama/Llama-Guard-3-1B + + +memory: + tests: + - memory/test_memory.py::test_query_documents + + scenarios: + - fixture_combo_id: ollama + - provider_fixtures: + inference: sentence_transformers + memory: faiss + - fixture_combo_id: chroma + + inference_models: + - meta-llama/Llama-3.2-1B-Instruct + + embedding_model: all-MiniLM-L6-v2 diff --git a/llama_stack/providers/tests/conftest.py b/llama_stack/providers/tests/conftest.py index 7408a6375..7d0d2ae74 100644 --- a/llama_stack/providers/tests/conftest.py +++ b/llama_stack/providers/tests/conftest.py @@ -5,18 +5,23 @@ # the root directory of this source tree. import os +from collections import defaultdict + from pathlib import Path from typing import Any, Dict, List, Optional import pytest +import yaml + from dotenv import load_dotenv -from pydantic import BaseModel +from pydantic import BaseModel, Field from termcolor import colored from llama_stack.distribution.datatypes import Provider from llama_stack.providers.datatypes import RemoteProviderConfig from .env import get_env_or_fail +from .report import Report class ProviderFixture(BaseModel): @@ -24,6 +29,83 @@ class ProviderFixture(BaseModel): provider_data: Optional[Dict[str, Any]] = None +class TestScenario(BaseModel): + # provider fixtures can be either a mark or a dictionary of api -> providers + provider_fixtures: Dict[str, str] = Field(default_factory=dict) + fixture_combo_id: Optional[str] = None + + +class APITestConfig(BaseModel): + scenarios: List[TestScenario] = Field(default_factory=list) + inference_models: List[str] = Field(default_factory=list) + + # test name format should be :: + tests: List[str] = Field(default_factory=list) + + +class MemoryApiTestConfig(APITestConfig): + embedding_model: Optional[str] = Field(default_factory=None) + + +class AgentsApiTestConfig(APITestConfig): + safety_shield: Optional[str] = Field(default_factory=None) + + +class TestConfig(BaseModel): + inference: Optional[APITestConfig] = None + agents: Optional[AgentsApiTestConfig] = None + memory: Optional[MemoryApiTestConfig] = None + + +def get_test_config_from_config_file(metafunc_config): + config_file = metafunc_config.getoption("--config") + if config_file is None: + return None + + config_file_path = Path(__file__).parent / config_file + if not config_file_path.exists(): + raise ValueError( + f"Test config {config_file} was specified but not found. Please make sure it exists in the llama_stack/providers/tests directory." + ) + with open(config_file_path, "r") as config_file: + config = yaml.safe_load(config_file) + return TestConfig(**config) + + +def get_test_config_for_api(metafunc_config, api): + test_config = get_test_config_from_config_file(metafunc_config) + if test_config is None: + return None + return getattr(test_config, api) + + +def get_provider_fixture_overrides_from_test_config( + metafunc_config, api, default_provider_fixture_combinations +): + api_config = get_test_config_for_api(metafunc_config, api) + if api_config is None: + return None + + fixture_combo_ids = set() + custom_provider_fixture_combos = [] + for scenario in api_config.scenarios: + if scenario.fixture_combo_id: + fixture_combo_ids.add(scenario.fixture_combo_id) + else: + custom_provider_fixture_combos.append( + pytest.param( + scenario.provider_fixtures, + id=scenario.provider_fixtures.get("inference") or "", + ) + ) + + if len(fixture_combo_ids) > 0: + for default_fixture in default_provider_fixture_combinations: + if default_fixture.id in fixture_combo_ids: + custom_provider_fixture_combos.append(default_fixture) + return custom_provider_fixture_combos + + def remote_stack_fixture() -> ProviderFixture: if url := os.getenv("REMOTE_STACK_URL", None): config = RemoteProviderConfig.from_url(url) @@ -59,6 +141,9 @@ def pytest_configure(config): key, value = env_var.split("=", 1) os.environ[key] = value + if config.getoption("--output") is not None: + config.pluginmanager.register(Report(config.getoption("--output"))) + def pytest_addoption(parser): parser.addoption( @@ -69,10 +154,44 @@ def pytest_addoption(parser): "Example: --providers inference=ollama,safety=meta-reference" ), ) + parser.addoption( + "--config", + action="store", + help="Set test config file (supported format: YAML), e.g. --config=test_config.yml", + ) + parser.addoption( + "--output", + action="store", + help="Set output file for test report, e.g. --output=pytest_report.md", + ) """Add custom command line options""" parser.addoption( "--env", action="append", help="Set environment variables, e.g. --env KEY=value" ) + parser.addoption( + "--inference-model", + action="store", + default="meta-llama/Llama-3.2-3B-Instruct", + help="Specify the inference model to use for testing", + ) + parser.addoption( + "--safety-shield", + action="store", + default="meta-llama/Llama-Guard-3-1B", + help="Specify the safety shield to use for testing", + ) + parser.addoption( + "--embedding-model", + action="store", + default=None, + help="Specify the embedding model to use for testing", + ) + parser.addoption( + "--judge-model", + action="store", + default="meta-llama/Llama-3.1-8B-Instruct", + help="Specify the judge model to use for testing", + ) def make_provider_id(providers: Dict[str, str]) -> str: @@ -148,10 +267,42 @@ def pytest_itemcollected(item): item.name = f"{item.name}[{marks}]" +def pytest_collection_modifyitems(session, config, items): + test_config = get_test_config_from_config_file(config) + if test_config is None: + return + + required_tests = defaultdict(set) + for api_test_config in [ + test_config.inference, + test_config.memory, + test_config.agents, + ]: + if api_test_config is None: + continue + for test in api_test_config.tests: + arr = test.split("::") + if len(arr) != 2: + raise ValueError(f"Invalid format for test name {test}") + test_path, func_name = arr + required_tests[Path(__file__).parent / test_path].add(func_name) + + new_items, deselected_items = [], [] + for item in items: + func_name = getattr(item, "originalname", item.name) + if func_name in required_tests[item.fspath]: + new_items.append(item) + continue + deselected_items.append(item) + + items[:] = new_items + config.hook.pytest_deselected(items=deselected_items) + + pytest_plugins = [ "llama_stack.providers.tests.inference.fixtures", "llama_stack.providers.tests.safety.fixtures", - "llama_stack.providers.tests.memory.fixtures", + "llama_stack.providers.tests.vector_io.fixtures", "llama_stack.providers.tests.agents.fixtures", "llama_stack.providers.tests.datasetio.fixtures", "llama_stack.providers.tests.scoring.fixtures", diff --git a/llama_stack/providers/tests/eval/conftest.py b/llama_stack/providers/tests/eval/conftest.py index 1bb49d41f..b7a68965e 100644 --- a/llama_stack/providers/tests/eval/conftest.py +++ b/llama_stack/providers/tests/eval/conftest.py @@ -15,6 +15,7 @@ from ..inference.fixtures import INFERENCE_FIXTURES from ..memory.fixtures import MEMORY_FIXTURES from ..safety.fixtures import SAFETY_FIXTURES from ..scoring.fixtures import SCORING_FIXTURES +from ..tools.fixtures import TOOL_RUNTIME_FIXTURES from .fixtures import EVAL_FIXTURES DEFAULT_PROVIDER_COMBINATIONS = [ @@ -27,6 +28,7 @@ DEFAULT_PROVIDER_COMBINATIONS = [ "agents": "meta_reference", "safety": "llama_guard", "memory": "faiss", + "tool_runtime": "memory_and_search", }, id="meta_reference_eval_fireworks_inference", marks=pytest.mark.meta_reference_eval_fireworks_inference, @@ -40,6 +42,7 @@ DEFAULT_PROVIDER_COMBINATIONS = [ "agents": "meta_reference", "safety": "llama_guard", "memory": "faiss", + "tool_runtime": "memory_and_search", }, id="meta_reference_eval_together_inference", marks=pytest.mark.meta_reference_eval_together_inference, @@ -53,6 +56,7 @@ DEFAULT_PROVIDER_COMBINATIONS = [ "agents": "meta_reference", "safety": "llama_guard", "memory": "faiss", + "tool_runtime": "memory_and_search", }, id="meta_reference_eval_together_inference_huggingface_datasetio", marks=pytest.mark.meta_reference_eval_together_inference_huggingface_datasetio, @@ -72,22 +76,6 @@ def pytest_configure(config): ) -def pytest_addoption(parser): - parser.addoption( - "--inference-model", - action="store", - default="meta-llama/Llama-3.2-3B-Instruct", - help="Specify the inference model to use for testing", - ) - - parser.addoption( - "--judge-model", - action="store", - default="meta-llama/Llama-3.1-8B-Instruct", - help="Specify the judge model to use for testing", - ) - - def pytest_generate_tests(metafunc): if "eval_stack" in metafunc.fixturenames: available_fixtures = { @@ -98,6 +86,7 @@ def pytest_generate_tests(metafunc): "agents": AGENTS_FIXTURES, "safety": SAFETY_FIXTURES, "memory": MEMORY_FIXTURES, + "tool_runtime": TOOL_RUNTIME_FIXTURES, } combinations = ( get_provider_fixture_overrides(metafunc.config, available_fixtures) diff --git a/llama_stack/providers/tests/eval/fixtures.py b/llama_stack/providers/tests/eval/fixtures.py index eba7c48a6..009e65fb3 100644 --- a/llama_stack/providers/tests/eval/fixtures.py +++ b/llama_stack/providers/tests/eval/fixtures.py @@ -35,7 +35,13 @@ EVAL_FIXTURES = ["meta_reference", "remote"] @pytest_asyncio.fixture(scope="session") -async def eval_stack(request, inference_model, judge_model): +async def eval_stack( + request, + inference_model, + judge_model, + tool_group_input_memory, + tool_group_input_tavily_search, +): fixture_dict = request.param providers = {} @@ -47,7 +53,8 @@ async def eval_stack(request, inference_model, judge_model): "inference", "agents", "safety", - "memory", + "vector_io", + "tool_runtime", ]: fixture = request.getfixturevalue(f"{key}_{fixture_dict[key]}") providers[key] = fixture.providers @@ -62,7 +69,8 @@ async def eval_stack(request, inference_model, judge_model): Api.scoring, Api.agents, Api.safety, - Api.memory, + Api.vector_io, + Api.tool_runtime, ], providers, provider_data, @@ -73,6 +81,7 @@ async def eval_stack(request, inference_model, judge_model): judge_model, ] ], + tool_groups=[tool_group_input_memory, tool_group_input_tavily_search], ) return test_stack.impls diff --git a/llama_stack/providers/tests/inference/conftest.py b/llama_stack/providers/tests/inference/conftest.py index 54ebcd83a..1303a1b35 100644 --- a/llama_stack/providers/tests/inference/conftest.py +++ b/llama_stack/providers/tests/inference/conftest.py @@ -6,26 +6,10 @@ import pytest -from ..conftest import get_provider_fixture_overrides - +from ..conftest import get_provider_fixture_overrides, get_test_config_for_api from .fixtures import INFERENCE_FIXTURES -def pytest_addoption(parser): - parser.addoption( - "--inference-model", - action="store", - default=None, - help="Specify the inference model to use for testing", - ) - parser.addoption( - "--embedding-model", - action="store", - default=None, - help="Specify the embedding model to use for testing", - ) - - def pytest_configure(config): for model in ["llama_8b", "llama_3b", "llama_vision"]: config.addinivalue_line( @@ -58,16 +42,21 @@ VISION_MODEL_PARAMS = [ def pytest_generate_tests(metafunc): + test_config = get_test_config_for_api(metafunc.config, "inference") + if "inference_model" in metafunc.fixturenames: - model = metafunc.config.getoption("--inference-model") - if model: + cls_name = metafunc.cls.__name__ + params = [] + inference_models = getattr(test_config, "inference_models", []) + for model in inference_models: + if ("Vision" in cls_name and "Vision" in model) or ( + "Vision" not in cls_name and "Vision" not in model + ): + params.append(pytest.param(model, id=model)) + + if not params: + model = metafunc.config.getoption("--inference-model") params = [pytest.param(model, id="")] - else: - cls_name = metafunc.cls.__name__ - if "Vision" in cls_name: - params = VISION_MODEL_PARAMS - else: - params = MODEL_PARAMS metafunc.parametrize( "inference_model", @@ -83,4 +72,13 @@ def pytest_generate_tests(metafunc): }, ): fixtures = [stack.values[0]["inference"] for stack in filtered_stacks] + if test_config: + if custom_fixtures := [ + ( + scenario.fixture_combo_id + or scenario.provider_fixtures.get("inference") + ) + for scenario in test_config.scenarios + ]: + fixtures = custom_fixtures metafunc.parametrize("inference_stack", fixtures, indirect=True) diff --git a/llama_stack/providers/tests/inference/fixtures.py b/llama_stack/providers/tests/inference/fixtures.py index 753d47f98..331898a7f 100644 --- a/llama_stack/providers/tests/inference/fixtures.py +++ b/llama_stack/providers/tests/inference/fixtures.py @@ -320,6 +320,7 @@ async def inference_stack(request, inference_model): inference_fixture.provider_data, models=[ ModelInput( + provider_id=inference_fixture.providers[0].provider_id, model_id=inference_model, model_type=model_type, metadata=metadata, diff --git a/llama_stack/providers/tests/inference/groq/test_groq_utils.py b/llama_stack/providers/tests/inference/groq/test_groq_utils.py index f3f263cb1..f6f593f16 100644 --- a/llama_stack/providers/tests/inference/groq/test_groq_utils.py +++ b/llama_stack/providers/tests/inference/groq/test_groq_utils.py @@ -21,6 +21,7 @@ from groq.types.chat.chat_completion_message_tool_call import ( Function, ) from groq.types.shared.function_definition import FunctionDefinition +from llama_models.datatypes import GreedySamplingStrategy, TopPSamplingStrategy from llama_models.llama3.api.datatypes import ToolParamDefinition from llama_stack.apis.inference import ( ChatCompletionRequest, @@ -152,21 +153,30 @@ class TestConvertChatCompletionRequest: assert converted["max_tokens"] == 100 - def test_includes_temperature(self): + def _dummy_chat_completion_request(self): + return ChatCompletionRequest( + model="Llama-3.2-3B", + messages=[UserMessage(content="Hello World")], + ) + + def test_includes_stratgy(self): request = self._dummy_chat_completion_request() - request.sampling_params.temperature = 0.5 + request.sampling_params.strategy = TopPSamplingStrategy( + temperature=0.5, top_p=0.95 + ) converted = convert_chat_completion_request(request) assert converted["temperature"] == 0.5 + assert converted["top_p"] == 0.95 - def test_includes_top_p(self): + def test_includes_greedy_strategy(self): request = self._dummy_chat_completion_request() - request.sampling_params.top_p = 0.95 + request.sampling_params.strategy = GreedySamplingStrategy() converted = convert_chat_completion_request(request) - assert converted["top_p"] == 0.95 + assert converted["temperature"] == 0.0 def test_includes_tool_choice(self): request = self._dummy_chat_completion_request() @@ -268,12 +278,6 @@ class TestConvertChatCompletionRequest: }, ] - def _dummy_chat_completion_request(self): - return ChatCompletionRequest( - model="Llama-3.2-3B", - messages=[UserMessage(content="Hello World")], - ) - class TestConvertNonStreamChatCompletionResponse: def test_returns_response(self): @@ -409,19 +413,19 @@ class TestConvertStreamChatCompletionResponse: iter = converted.__aiter__() chunk = await iter.__anext__() assert chunk.event.event_type == ChatCompletionResponseEventType.start - assert chunk.event.delta == "Hello " + assert chunk.event.delta.text == "Hello " chunk = await iter.__anext__() assert chunk.event.event_type == ChatCompletionResponseEventType.progress - assert chunk.event.delta == "World " + assert chunk.event.delta.text == "World " chunk = await iter.__anext__() assert chunk.event.event_type == ChatCompletionResponseEventType.progress - assert chunk.event.delta == " !" + assert chunk.event.delta.text == " !" chunk = await iter.__anext__() assert chunk.event.event_type == ChatCompletionResponseEventType.complete - assert chunk.event.delta == "" + assert chunk.event.delta.text == "" assert chunk.event.stop_reason == StopReason.end_of_turn with pytest.raises(StopAsyncIteration): @@ -468,7 +472,7 @@ class TestConvertStreamChatCompletionResponse: iter = converted.__aiter__() chunk = await iter.__anext__() assert chunk.event.event_type == ChatCompletionResponseEventType.start - assert chunk.event.delta.content == ToolCall( + assert chunk.event.delta.tool_call == ToolCall( call_id="tool_call_id", tool_name="get_flight_info", arguments={"origin": "AU", "destination": "LAX"}, diff --git a/llama_stack/providers/tests/inference/test_text_inference.py b/llama_stack/providers/tests/inference/test_text_inference.py index 19bd30ec7..7201fdc4a 100644 --- a/llama_stack/providers/tests/inference/test_text_inference.py +++ b/llama_stack/providers/tests/inference/test_text_inference.py @@ -31,7 +31,8 @@ from llama_stack.apis.inference import ( ToolChoice, UserMessage, ) -from llama_stack.apis.models import Model +from llama_stack.apis.models import ListModelsResponse, Model + from .utils import group_chunks @@ -91,12 +92,13 @@ class TestInference: async def test_model_list(self, inference_model, inference_stack): _, models_impl = inference_stack response = await models_impl.list_models() - assert isinstance(response, list) - assert len(response) >= 1 - assert all(isinstance(model, Model) for model in response) + assert isinstance(response, ListModelsResponse) + assert isinstance(response.data, list) + assert len(response.data) >= 1 + assert all(isinstance(model, Model) for model in response.data) model_def = None - for model in response: + for model in response.data: if model.identifier == inference_model: model_def = model break @@ -116,6 +118,7 @@ class TestInference: "remote::fireworks", "remote::nvidia", "remote::cerebras", + "remote::vllm", ): pytest.skip("Other inference providers don't support completion() yet") @@ -206,7 +209,6 @@ class TestInference: assert not chunk.logprobs, "Logprobs should be empty" @pytest.mark.asyncio(loop_scope="session") - @pytest.mark.skip("This test is not quite robust") async def test_completion_structured_output(self, inference_model, inference_stack): inference_impl, _ = inference_stack @@ -478,16 +480,16 @@ class TestInference: ) first = grouped[ChatCompletionResponseEventType.progress][0] if not isinstance( - first.event.delta.content, ToolCall + first.event.delta.tool_call, ToolCall ): # first chunk may contain entire call assert first.event.delta.parse_status == ToolCallParseStatus.started last = grouped[ChatCompletionResponseEventType.progress][-1] # assert last.event.stop_reason == expected_stop_reason assert last.event.delta.parse_status == ToolCallParseStatus.succeeded - assert last.event.delta.content.type == "tool_call" + assert isinstance(last.event.delta.tool_call, ToolCall) - call = last.event.delta.content + call = last.event.delta.tool_call assert call.tool_name == "get_weather" assert "location" in call.arguments assert "San Francisco" in call.arguments["location"] diff --git a/llama_stack/providers/tests/inference/test_vision_inference.py b/llama_stack/providers/tests/inference/test_vision_inference.py index 6374310f3..fba7cefde 100644 --- a/llama_stack/providers/tests/inference/test_vision_inference.py +++ b/llama_stack/providers/tests/inference/test_vision_inference.py @@ -32,13 +32,15 @@ class TestVisionModelInference: "image, expected_strings", [ ( - ImageContentItem(data=PASTA_IMAGE), + ImageContentItem(image=dict(data=PASTA_IMAGE)), ["spaghetti"], ), ( ImageContentItem( - url=URL( - uri="https://www.healthypawspetinsurance.com/Images/V3/DogAndPuppyInsurance/Dog_CTA_Desktop_HeroImage.jpg" + image=dict( + url=URL( + uri="https://www.healthypawspetinsurance.com/Images/V3/DogAndPuppyInsurance/Dog_CTA_Desktop_HeroImage.jpg" + ) ) ), ["puppy"], @@ -105,8 +107,10 @@ class TestVisionModelInference: images = [ ImageContentItem( - url=URL( - uri="https://www.healthypawspetinsurance.com/Images/V3/DogAndPuppyInsurance/Dog_CTA_Desktop_HeroImage.jpg" + image=dict( + url=URL( + uri="https://www.healthypawspetinsurance.com/Images/V3/DogAndPuppyInsurance/Dog_CTA_Desktop_HeroImage.jpg" + ) ) ), ] diff --git a/llama_stack/providers/tests/memory/test_memory.py b/llama_stack/providers/tests/memory/test_memory.py deleted file mode 100644 index 801b04dfc..000000000 --- a/llama_stack/providers/tests/memory/test_memory.py +++ /dev/null @@ -1,192 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the terms described in the LICENSE file in -# the root directory of this source tree. - -import uuid - -import pytest - -from llama_stack.apis.memory import MemoryBankDocument, QueryDocumentsResponse - -from llama_stack.apis.memory_banks import ( - MemoryBank, - MemoryBanks, - VectorMemoryBankParams, -) - -# How to run this test: -# -# pytest llama_stack/providers/tests/memory/test_memory.py -# -m "sentence_transformers" --env EMBEDDING_DIMENSION=384 -# -v -s --tb=short --disable-warnings - - -@pytest.fixture -def sample_documents(): - return [ - MemoryBankDocument( - document_id="doc1", - content="Python is a high-level programming language.", - metadata={"category": "programming", "difficulty": "beginner"}, - ), - MemoryBankDocument( - document_id="doc2", - content="Machine learning is a subset of artificial intelligence.", - metadata={"category": "AI", "difficulty": "advanced"}, - ), - MemoryBankDocument( - document_id="doc3", - content="Data structures are fundamental to computer science.", - metadata={"category": "computer science", "difficulty": "intermediate"}, - ), - MemoryBankDocument( - document_id="doc4", - content="Neural networks are inspired by biological neural networks.", - metadata={"category": "AI", "difficulty": "advanced"}, - ), - ] - - -async def register_memory_bank( - banks_impl: MemoryBanks, embedding_model: str -) -> MemoryBank: - bank_id = f"test_bank_{uuid.uuid4().hex}" - return await banks_impl.register_memory_bank( - memory_bank_id=bank_id, - params=VectorMemoryBankParams( - embedding_model=embedding_model, - chunk_size_in_tokens=512, - overlap_size_in_tokens=64, - ), - ) - - -class TestMemory: - @pytest.mark.asyncio - async def test_banks_list(self, memory_stack, embedding_model): - _, banks_impl = memory_stack - - # Register a test bank - registered_bank = await register_memory_bank(banks_impl, embedding_model) - - try: - # Verify our bank shows up in list - response = await banks_impl.list_memory_banks() - assert isinstance(response, list) - assert any( - bank.memory_bank_id == registered_bank.memory_bank_id - for bank in response - ) - finally: - # Clean up - await banks_impl.unregister_memory_bank(registered_bank.memory_bank_id) - - # Verify our bank was removed - response = await banks_impl.list_memory_banks() - assert all( - bank.memory_bank_id != registered_bank.memory_bank_id for bank in response - ) - - @pytest.mark.asyncio - async def test_banks_register(self, memory_stack, embedding_model): - _, banks_impl = memory_stack - - bank_id = f"test_bank_{uuid.uuid4().hex}" - - try: - # Register initial bank - await banks_impl.register_memory_bank( - memory_bank_id=bank_id, - params=VectorMemoryBankParams( - embedding_model=embedding_model, - chunk_size_in_tokens=512, - overlap_size_in_tokens=64, - ), - ) - - # Verify our bank exists - response = await banks_impl.list_memory_banks() - assert isinstance(response, list) - assert any(bank.memory_bank_id == bank_id for bank in response) - - # Try registering same bank again - await banks_impl.register_memory_bank( - memory_bank_id=bank_id, - params=VectorMemoryBankParams( - embedding_model=embedding_model, - chunk_size_in_tokens=512, - overlap_size_in_tokens=64, - ), - ) - - # Verify still only one instance of our bank - response = await banks_impl.list_memory_banks() - assert isinstance(response, list) - assert ( - len([bank for bank in response if bank.memory_bank_id == bank_id]) == 1 - ) - finally: - # Clean up - await banks_impl.unregister_memory_bank(bank_id) - - @pytest.mark.asyncio - async def test_query_documents( - self, memory_stack, embedding_model, sample_documents - ): - memory_impl, banks_impl = memory_stack - - with pytest.raises(ValueError): - await memory_impl.insert_documents("test_bank", sample_documents) - - registered_bank = await register_memory_bank(banks_impl, embedding_model) - await memory_impl.insert_documents( - registered_bank.memory_bank_id, sample_documents - ) - - query1 = "programming language" - response1 = await memory_impl.query_documents( - registered_bank.memory_bank_id, query1 - ) - assert_valid_response(response1) - assert any("Python" in chunk.content for chunk in response1.chunks) - - # Test case 3: Query with semantic similarity - query3 = "AI and brain-inspired computing" - response3 = await memory_impl.query_documents( - registered_bank.memory_bank_id, query3 - ) - assert_valid_response(response3) - assert any( - "neural networks" in chunk.content.lower() for chunk in response3.chunks - ) - - # Test case 4: Query with limit on number of results - query4 = "computer" - params4 = {"max_chunks": 2} - response4 = await memory_impl.query_documents( - registered_bank.memory_bank_id, query4, params4 - ) - assert_valid_response(response4) - assert len(response4.chunks) <= 2 - - # Test case 5: Query with threshold on similarity score - query5 = "quantum computing" # Not directly related to any document - params5 = {"score_threshold": 0.01} - response5 = await memory_impl.query_documents( - registered_bank.memory_bank_id, query5, params5 - ) - assert_valid_response(response5) - print("The scores are:", response5.scores) - assert all(score >= 0.01 for score in response5.scores) - - -def assert_valid_response(response: QueryDocumentsResponse): - assert isinstance(response, QueryDocumentsResponse) - assert len(response.chunks) > 0 - assert len(response.scores) > 0 - assert len(response.chunks) == len(response.scores) - for chunk in response.chunks: - assert isinstance(chunk.content, str) - assert chunk.document_id is not None diff --git a/llama_stack/providers/tests/post_training/test_post_training.py b/llama_stack/providers/tests/post_training/test_post_training.py index 0645cd555..0c58c1fa0 100644 --- a/llama_stack/providers/tests/post_training/test_post_training.py +++ b/llama_stack/providers/tests/post_training/test_post_training.py @@ -5,7 +5,7 @@ # the root directory of this source tree. import pytest -from llama_stack.apis.common.type_system import JobStatus +from llama_stack.apis.common.job_types import JobStatus from llama_stack.apis.post_training import ( Checkpoint, DataConfig, diff --git a/llama_stack/providers/tests/report.py b/llama_stack/providers/tests/report.py new file mode 100644 index 000000000..c07d7278a --- /dev/null +++ b/llama_stack/providers/tests/report.py @@ -0,0 +1,200 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + + +from collections import defaultdict +from pathlib import Path + +import pytest +from llama_models.datatypes import CoreModelId +from llama_models.sku_list import all_registered_models +from pytest import ExitCode + +from pytest_html.basereport import _process_outcome + + +INFERENCE_APIS = ["chat_completion"] +FUNCTIONALITIES = ["streaming", "structured_output", "tool_calling"] +SUPPORTED_MODELS = { + "ollama": set( + [ + CoreModelId.llama3_1_8b_instruct.value, + CoreModelId.llama3_1_8b_instruct.value, + CoreModelId.llama3_1_70b_instruct.value, + CoreModelId.llama3_1_70b_instruct.value, + CoreModelId.llama3_1_405b_instruct.value, + CoreModelId.llama3_1_405b_instruct.value, + CoreModelId.llama3_2_1b_instruct.value, + CoreModelId.llama3_2_1b_instruct.value, + CoreModelId.llama3_2_3b_instruct.value, + CoreModelId.llama3_2_3b_instruct.value, + CoreModelId.llama3_2_11b_vision_instruct.value, + CoreModelId.llama3_2_11b_vision_instruct.value, + CoreModelId.llama3_2_90b_vision_instruct.value, + CoreModelId.llama3_2_90b_vision_instruct.value, + CoreModelId.llama3_3_70b_instruct.value, + CoreModelId.llama_guard_3_8b.value, + CoreModelId.llama_guard_3_1b.value, + ] + ), + "fireworks": set( + [ + CoreModelId.llama3_1_8b_instruct.value, + CoreModelId.llama3_1_70b_instruct.value, + CoreModelId.llama3_1_405b_instruct.value, + CoreModelId.llama3_2_1b_instruct.value, + CoreModelId.llama3_2_3b_instruct.value, + CoreModelId.llama3_2_11b_vision_instruct.value, + CoreModelId.llama3_2_90b_vision_instruct.value, + CoreModelId.llama3_3_70b_instruct.value, + CoreModelId.llama_guard_3_8b.value, + CoreModelId.llama_guard_3_11b_vision.value, + ] + ), + "together": set( + [ + CoreModelId.llama3_1_8b_instruct.value, + CoreModelId.llama3_1_70b_instruct.value, + CoreModelId.llama3_1_405b_instruct.value, + CoreModelId.llama3_2_3b_instruct.value, + CoreModelId.llama3_2_11b_vision_instruct.value, + CoreModelId.llama3_2_90b_vision_instruct.value, + CoreModelId.llama3_3_70b_instruct.value, + CoreModelId.llama_guard_3_8b.value, + CoreModelId.llama_guard_3_11b_vision.value, + ] + ), +} + + +class Report: + + def __init__(self, output_path): + + valid_file_format = ( + output_path.split(".")[1] in ["md", "markdown"] + if len(output_path.split(".")) == 2 + else False + ) + if not valid_file_format: + raise ValueError( + f"Invalid output file {output_path}. Markdown file is required" + ) + self.output_path = output_path + self.test_data = defaultdict(dict) + self.inference_tests = defaultdict(dict) + + @pytest.hookimpl + def pytest_runtest_logreport(self, report): + # This hook is called in several phases, including setup, call and teardown + # The test is considered failed / error if any of the outcomes is not "Passed" + outcome = _process_outcome(report) + data = { + "outcome": report.outcome, + "longrepr": report.longrepr, + "name": report.nodeid, + } + if report.nodeid not in self.test_data: + self.test_data[report.nodeid] = data + elif self.test_data[report.nodeid] != outcome and outcome != "Passed": + self.test_data[report.nodeid] = data + + @pytest.hookimpl + def pytest_sessionfinish(self, session, exitstatus): + if exitstatus <= ExitCode.INTERRUPTED: + return + report = [] + report.append("# Llama Stack Integration Test Results Report") + report.append("\n## Summary") + report.append("\n## Supported Models: ") + + header = "| Model Descriptor |" + dividor = "|:---|" + for k in SUPPORTED_MODELS.keys(): + header += f"{k} |" + dividor += ":---:|" + + report.append(header) + report.append(dividor) + + rows = [] + for model in all_registered_models(): + if ( + "Instruct" not in model.core_model_id.value + and "Guard" not in model.core_model_id.value + ): + continue + row = f"| {model.core_model_id.value} |" + for k in SUPPORTED_MODELS.keys(): + if model.core_model_id.value in SUPPORTED_MODELS[k]: + row += " ✅ |" + else: + row += " ❌ |" + rows.append(row) + report.extend(rows) + + report.append("\n### Tests:") + + for provider in SUPPORTED_MODELS.keys(): + if provider not in self.inference_tests: + continue + report.append(f"\n #### {provider}") + test_table = [ + "| Area | Model | API | Functionality Test | Status |", + "|:-----|:-----|:-----|:-----|:-----|", + ] + for api in INFERENCE_APIS: + tests = self.inference_tests[provider][api] + for test_nodeid in tests: + row = "|{area} | {model} | {api} | {test} | {result} ".format( + area="Text" if "text" in test_nodeid else "Vision", + model=( + "Llama-3.1-8B-Instruct" + if "text" in test_nodeid + else "Llama3.2-11B-Vision-Instruct" + ), + api=f"/{api}", + test=self.get_simple_function_name(test_nodeid), + result=( + "✅" + if self.test_data[test_nodeid]["outcome"] == "passed" + else "❌" + ), + ) + test_table += [row] + report.extend(test_table) + report.append("\n") + + output_file = Path(self.output_path) + output_file.write_text("\n".join(report)) + print(f"\n Report generated: {output_file.absolute()}") + + @pytest.hookimpl(trylast=True) + def pytest_collection_modifyitems(self, session, config, items): + for item in items: + inference = item.callspec.params.get("inference_stack") + if "inference" in item.nodeid: + func_name = getattr(item, "originalname", item.name) + for api in INFERENCE_APIS: + if api in func_name: + api_tests = self.inference_tests[inference].get(api, set()) + api_tests.add(item.nodeid) + self.inference_tests[inference][api] = api_tests + + def get_simple_function_name(self, nodeid): + """Extract function name from nodeid. + + Examples: + - 'tests/test_math.py::test_addition' -> 'test_addition' + - 'tests/test_math.py::TestClass::test_method' -> test_method' + """ + parts = nodeid.split("::") + func_name = nodeid # Fallback to full nodeid if pattern doesn't match + if len(parts) == 2: # Simple function + func_name = parts[1] + elif len(parts) == 3: # Class method + func_name = parts[2] + return func_name.split("[")[0] diff --git a/llama_stack/providers/tests/resolver.py b/llama_stack/providers/tests/resolver.py index 81816d51e..f0c4c530e 100644 --- a/llama_stack/providers/tests/resolver.py +++ b/llama_stack/providers/tests/resolver.py @@ -12,11 +12,11 @@ from pydantic import BaseModel from llama_stack.apis.datasets import DatasetInput from llama_stack.apis.eval_tasks import EvalTaskInput -from llama_stack.apis.memory_banks import MemoryBankInput from llama_stack.apis.models import ModelInput from llama_stack.apis.scoring_functions import ScoringFnInput from llama_stack.apis.shields import ShieldInput from llama_stack.apis.tools import ToolGroupInput +from llama_stack.apis.vector_dbs import VectorDBInput from llama_stack.distribution.build import print_pip_install_help from llama_stack.distribution.configure import parse_and_maybe_upgrade_config from llama_stack.distribution.datatypes import Provider, StackRunConfig @@ -39,7 +39,7 @@ async def construct_stack_for_test( provider_data: Optional[Dict[str, Any]] = None, models: Optional[List[ModelInput]] = None, shields: Optional[List[ShieldInput]] = None, - memory_banks: Optional[List[MemoryBankInput]] = None, + vector_dbs: Optional[List[VectorDBInput]] = None, datasets: Optional[List[DatasetInput]] = None, scoring_fns: Optional[List[ScoringFnInput]] = None, eval_tasks: Optional[List[EvalTaskInput]] = None, @@ -53,7 +53,7 @@ async def construct_stack_for_test( metadata_store=SqliteKVStoreConfig(db_path=sqlite_file.name), models=models or [], shields=shields or [], - memory_banks=memory_banks or [], + vector_dbs=vector_dbs or [], datasets=datasets or [], scoring_fns=scoring_fns or [], eval_tasks=eval_tasks or [], diff --git a/llama_stack/providers/tests/safety/conftest.py b/llama_stack/providers/tests/safety/conftest.py index 6846517e3..a5e77f570 100644 --- a/llama_stack/providers/tests/safety/conftest.py +++ b/llama_stack/providers/tests/safety/conftest.py @@ -64,15 +64,6 @@ def pytest_configure(config): ) -def pytest_addoption(parser): - parser.addoption( - "--safety-shield", - action="store", - default=None, - help="Specify the safety shield to use for testing", - ) - - SAFETY_SHIELD_PARAMS = [ pytest.param( "meta-llama/Llama-Guard-3-1B", marks=pytest.mark.guard_1b, id="guard_1b" diff --git a/llama_stack/providers/tests/scoring/conftest.py b/llama_stack/providers/tests/scoring/conftest.py index dc4979dd7..0b4e7d46e 100644 --- a/llama_stack/providers/tests/scoring/conftest.py +++ b/llama_stack/providers/tests/scoring/conftest.py @@ -55,21 +55,6 @@ def pytest_configure(config): ) -def pytest_addoption(parser): - parser.addoption( - "--inference-model", - action="store", - default="meta-llama/Llama-3.2-3B-Instruct", - help="Specify the inference model to use for testing", - ) - parser.addoption( - "--judge-model", - action="store", - default="meta-llama/Llama-3.1-8B-Instruct", - help="Specify the judge model to use for testing", - ) - - def pytest_generate_tests(metafunc): judge_model = metafunc.config.getoption("--judge-model") if "judge_model" in metafunc.fixturenames: diff --git a/llama_stack/providers/tests/tools/conftest.py b/llama_stack/providers/tests/tools/conftest.py index 11aad5ab6..0df547a9d 100644 --- a/llama_stack/providers/tests/tools/conftest.py +++ b/llama_stack/providers/tests/tools/conftest.py @@ -8,8 +8,8 @@ import pytest from ..conftest import get_provider_fixture_overrides from ..inference.fixtures import INFERENCE_FIXTURES -from ..memory.fixtures import MEMORY_FIXTURES from ..safety.fixtures import SAFETY_FIXTURES +from ..vector_io.fixtures import VECTOR_IO_FIXTURES from .fixtures import TOOL_RUNTIME_FIXTURES DEFAULT_PROVIDER_COMBINATIONS = [ @@ -17,7 +17,7 @@ DEFAULT_PROVIDER_COMBINATIONS = [ { "inference": "together", "safety": "llama_guard", - "memory": "faiss", + "vector_io": "faiss", "tool_runtime": "memory_and_search", }, id="together", @@ -34,32 +34,16 @@ def pytest_configure(config): ) -def pytest_addoption(parser): - parser.addoption( - "--inference-model", - action="store", - default="meta-llama/Llama-3.2-3B-Instruct", - help="Specify the inference model to use for testing", - ) - parser.addoption( - "--safety-shield", - action="store", - default="meta-llama/Llama-Guard-3-1B", - help="Specify the safety shield to use for testing", - ) - - def pytest_generate_tests(metafunc): if "tools_stack" in metafunc.fixturenames: available_fixtures = { "inference": INFERENCE_FIXTURES, "safety": SAFETY_FIXTURES, - "memory": MEMORY_FIXTURES, + "vector_io": VECTOR_IO_FIXTURES, "tool_runtime": TOOL_RUNTIME_FIXTURES, } combinations = ( get_provider_fixture_overrides(metafunc.config, available_fixtures) or DEFAULT_PROVIDER_COMBINATIONS ) - print(combinations) metafunc.parametrize("tools_stack", combinations, indirect=True) diff --git a/llama_stack/providers/tests/tools/fixtures.py b/llama_stack/providers/tests/tools/fixtures.py index a559dbf8c..a2dd4239a 100644 --- a/llama_stack/providers/tests/tools/fixtures.py +++ b/llama_stack/providers/tests/tools/fixtures.py @@ -22,8 +22,8 @@ def tool_runtime_memory_and_search() -> ProviderFixture: return ProviderFixture( providers=[ Provider( - provider_id="memory-runtime", - provider_type="inline::memory-runtime", + provider_id="rag-runtime", + provider_type="inline::rag-runtime", config={}, ), Provider( @@ -47,8 +47,8 @@ def tool_runtime_memory_and_search() -> ProviderFixture: @pytest.fixture(scope="session") def tool_group_input_memory() -> ToolGroupInput: return ToolGroupInput( - toolgroup_id="builtin::memory", - provider_id="memory-runtime", + toolgroup_id="builtin::rag", + provider_id="rag-runtime", ) @@ -83,7 +83,7 @@ async def tools_stack( providers = {} provider_data = {} - for key in ["inference", "memory", "tool_runtime"]: + for key in ["inference", "vector_io", "tool_runtime"]: fixture = request.getfixturevalue(f"{key}_{fixture_dict[key]}") providers[key] = fixture.providers if key == "inference": @@ -117,7 +117,12 @@ async def tools_stack( ) test_stack = await construct_stack_for_test( - [Api.tool_groups, Api.inference, Api.memory, Api.tool_runtime], + [ + Api.tool_groups, + Api.inference, + Api.vector_io, + Api.tool_runtime, + ], providers, provider_data, models=models, diff --git a/llama_stack/providers/tests/tools/test_tools.py b/llama_stack/providers/tests/tools/test_tools.py index 16081b939..281ea404d 100644 --- a/llama_stack/providers/tests/tools/test_tools.py +++ b/llama_stack/providers/tests/tools/test_tools.py @@ -8,10 +8,7 @@ import os import pytest -from llama_stack.apis.inference import UserMessage -from llama_stack.apis.memory import MemoryBankDocument -from llama_stack.apis.memory_banks import VectorMemoryBankParams -from llama_stack.apis.tools import ToolInvocationResult +from llama_stack.apis.tools import RAGDocument, RAGQueryResult, ToolInvocationResult from llama_stack.providers.datatypes import Api @@ -36,7 +33,7 @@ def sample_documents(): "lora_finetune.rst", ] return [ - MemoryBankDocument( + RAGDocument( document_id=f"num-{i}", content=f"https://raw.githubusercontent.com/pytorch/torchtune/main/docs/source/tutorials/{url}", mime_type="text/plain", @@ -57,7 +54,7 @@ class TestTools: # Execute the tool response = await tools_impl.invoke_tool( - tool_name="web_search", args={"query": sample_search_query} + tool_name="web_search", kwargs={"query": sample_search_query} ) # Verify the response @@ -75,7 +72,7 @@ class TestTools: tools_impl = tools_stack.impls[Api.tool_runtime] response = await tools_impl.invoke_tool( - tool_name="wolfram_alpha", args={"query": sample_wolfram_alpha_query} + tool_name="wolfram_alpha", kwargs={"query": sample_wolfram_alpha_query} ) # Verify the response @@ -85,43 +82,33 @@ class TestTools: assert isinstance(response.content, str) @pytest.mark.asyncio - async def test_memory_tool(self, tools_stack, sample_documents): + async def test_rag_tool(self, tools_stack, sample_documents): """Test the memory tool functionality.""" - memory_banks_impl = tools_stack.impls[Api.memory_banks] - memory_impl = tools_stack.impls[Api.memory] + vector_dbs_impl = tools_stack.impls[Api.vector_dbs] tools_impl = tools_stack.impls[Api.tool_runtime] # Register memory bank - await memory_banks_impl.register_memory_bank( - memory_bank_id="test_bank", - params=VectorMemoryBankParams( - embedding_model="all-MiniLM-L6-v2", - chunk_size_in_tokens=512, - overlap_size_in_tokens=64, - ), + await vector_dbs_impl.register_vector_db( + vector_db_id="test_bank", + embedding_model="all-MiniLM-L6-v2", + embedding_dimension=384, provider_id="faiss", ) # Insert documents into memory - await memory_impl.insert_documents( - bank_id="test_bank", + await tools_impl.rag_tool.insert( documents=sample_documents, + vector_db_id="test_bank", + chunk_size_in_tokens=512, ) # Execute the memory tool - response = await tools_impl.invoke_tool( - tool_name="memory", - args={ - "messages": [ - UserMessage( - content="What are the main topics covered in the documentation?", - ) - ], - "memory_bank_ids": ["test_bank"], - }, + response = await tools_impl.rag_tool.query( + content="What are the main topics covered in the documentation?", + vector_db_ids=["test_bank"], ) # Verify the response - assert isinstance(response, ToolInvocationResult) + assert isinstance(response, RAGQueryResult) assert response.content is not None assert len(response.content) > 0 diff --git a/tests/client-sdk/memory/__init__.py b/llama_stack/providers/tests/vector_io/__init__.py similarity index 100% rename from tests/client-sdk/memory/__init__.py rename to llama_stack/providers/tests/vector_io/__init__.py diff --git a/llama_stack/providers/tests/memory/conftest.py b/llama_stack/providers/tests/vector_io/conftest.py similarity index 61% rename from llama_stack/providers/tests/memory/conftest.py rename to llama_stack/providers/tests/vector_io/conftest.py index 9b6ba177d..df5c8ea6a 100644 --- a/llama_stack/providers/tests/memory/conftest.py +++ b/llama_stack/providers/tests/vector_io/conftest.py @@ -6,17 +6,21 @@ import pytest -from ..conftest import get_provider_fixture_overrides +from ..conftest import ( + get_provider_fixture_overrides, + get_provider_fixture_overrides_from_test_config, + get_test_config_for_api, +) from ..inference.fixtures import INFERENCE_FIXTURES -from .fixtures import MEMORY_FIXTURES +from .fixtures import VECTOR_IO_FIXTURES DEFAULT_PROVIDER_COMBINATIONS = [ pytest.param( { "inference": "sentence_transformers", - "memory": "faiss", + "vector_io": "faiss", }, id="sentence_transformers", marks=pytest.mark.sentence_transformers, @@ -24,7 +28,7 @@ DEFAULT_PROVIDER_COMBINATIONS = [ pytest.param( { "inference": "ollama", - "memory": "faiss", + "vector_io": "faiss", }, id="ollama", marks=pytest.mark.ollama, @@ -32,7 +36,7 @@ DEFAULT_PROVIDER_COMBINATIONS = [ pytest.param( { "inference": "sentence_transformers", - "memory": "chroma", + "vector_io": "chroma", }, id="chroma", marks=pytest.mark.chroma, @@ -40,7 +44,7 @@ DEFAULT_PROVIDER_COMBINATIONS = [ pytest.param( { "inference": "bedrock", - "memory": "qdrant", + "vector_io": "qdrant", }, id="qdrant", marks=pytest.mark.qdrant, @@ -48,7 +52,7 @@ DEFAULT_PROVIDER_COMBINATIONS = [ pytest.param( { "inference": "fireworks", - "memory": "weaviate", + "vector_io": "weaviate", }, id="weaviate", marks=pytest.mark.weaviate, @@ -56,17 +60,8 @@ DEFAULT_PROVIDER_COMBINATIONS = [ ] -def pytest_addoption(parser): - parser.addoption( - "--embedding-model", - action="store", - default=None, - help="Specify the embedding model to use for testing", - ) - - def pytest_configure(config): - for fixture_name in MEMORY_FIXTURES: + for fixture_name in VECTOR_IO_FIXTURES: config.addinivalue_line( "markers", f"{fixture_name}: marks tests as {fixture_name} specific", @@ -74,8 +69,11 @@ def pytest_configure(config): def pytest_generate_tests(metafunc): + test_config = get_test_config_for_api(metafunc.config, "vector_io") if "embedding_model" in metafunc.fixturenames: - model = metafunc.config.getoption("--embedding-model") + model = getattr(test_config, "embedding_model", None) + # Fall back to the default if not specified by the config file + model = model or metafunc.config.getoption("--embedding-model") if model: params = [pytest.param(model, id="")] else: @@ -83,13 +81,16 @@ def pytest_generate_tests(metafunc): metafunc.parametrize("embedding_model", params, indirect=True) - if "memory_stack" in metafunc.fixturenames: + if "vector_io_stack" in metafunc.fixturenames: available_fixtures = { "inference": INFERENCE_FIXTURES, - "memory": MEMORY_FIXTURES, + "vector_io": VECTOR_IO_FIXTURES, } combinations = ( - get_provider_fixture_overrides(metafunc.config, available_fixtures) + get_provider_fixture_overrides_from_test_config( + metafunc.config, "vector_io", DEFAULT_PROVIDER_COMBINATIONS + ) + or get_provider_fixture_overrides(metafunc.config, available_fixtures) or DEFAULT_PROVIDER_COMBINATIONS ) - metafunc.parametrize("memory_stack", combinations, indirect=True) + metafunc.parametrize("vector_io_stack", combinations, indirect=True) diff --git a/llama_stack/providers/tests/memory/fixtures.py b/llama_stack/providers/tests/vector_io/fixtures.py similarity index 80% rename from llama_stack/providers/tests/memory/fixtures.py rename to llama_stack/providers/tests/vector_io/fixtures.py index b9dbb84f7..c8d5fa8cf 100644 --- a/llama_stack/providers/tests/memory/fixtures.py +++ b/llama_stack/providers/tests/vector_io/fixtures.py @@ -12,11 +12,12 @@ import pytest_asyncio from llama_stack.apis.models import ModelInput, ModelType from llama_stack.distribution.datatypes import Api, Provider -from llama_stack.providers.inline.memory.chroma import ChromaInlineImplConfig -from llama_stack.providers.inline.memory.faiss import FaissImplConfig -from llama_stack.providers.remote.memory.chroma import ChromaRemoteImplConfig -from llama_stack.providers.remote.memory.pgvector import PGVectorConfig -from llama_stack.providers.remote.memory.weaviate import WeaviateConfig + +from llama_stack.providers.inline.vector_io.chroma import ChromaInlineImplConfig +from llama_stack.providers.inline.vector_io.faiss import FaissImplConfig +from llama_stack.providers.remote.vector_io.chroma import ChromaRemoteImplConfig +from llama_stack.providers.remote.vector_io.pgvector import PGVectorConfig +from llama_stack.providers.remote.vector_io.weaviate import WeaviateConfig from llama_stack.providers.tests.resolver import construct_stack_for_test from llama_stack.providers.utils.kvstore.config import SqliteKVStoreConfig @@ -32,12 +33,12 @@ def embedding_model(request): @pytest.fixture(scope="session") -def memory_remote() -> ProviderFixture: +def vector_io_remote() -> ProviderFixture: return remote_stack_fixture() @pytest.fixture(scope="session") -def memory_faiss() -> ProviderFixture: +def vector_io_faiss() -> ProviderFixture: temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".db") return ProviderFixture( providers=[ @@ -53,7 +54,7 @@ def memory_faiss() -> ProviderFixture: @pytest.fixture(scope="session") -def memory_pgvector() -> ProviderFixture: +def vector_io_pgvector() -> ProviderFixture: return ProviderFixture( providers=[ Provider( @@ -72,7 +73,7 @@ def memory_pgvector() -> ProviderFixture: @pytest.fixture(scope="session") -def memory_weaviate() -> ProviderFixture: +def vector_io_weaviate() -> ProviderFixture: return ProviderFixture( providers=[ Provider( @@ -89,7 +90,7 @@ def memory_weaviate() -> ProviderFixture: @pytest.fixture(scope="session") -def memory_chroma() -> ProviderFixture: +def vector_io_chroma() -> ProviderFixture: url = os.getenv("CHROMA_URL") if url: config = ChromaRemoteImplConfig(url=url) @@ -110,23 +111,23 @@ def memory_chroma() -> ProviderFixture: ) -MEMORY_FIXTURES = ["faiss", "pgvector", "weaviate", "remote", "chroma"] +VECTOR_IO_FIXTURES = ["faiss", "pgvector", "weaviate", "chroma"] @pytest_asyncio.fixture(scope="session") -async def memory_stack(embedding_model, request): +async def vector_io_stack(embedding_model, request): fixture_dict = request.param providers = {} provider_data = {} - for key in ["inference", "memory"]: + for key in ["inference", "vector_io"]: fixture = request.getfixturevalue(f"{key}_{fixture_dict[key]}") providers[key] = fixture.providers if fixture.provider_data: provider_data.update(fixture.provider_data) test_stack = await construct_stack_for_test( - [Api.memory, Api.inference], + [Api.vector_io, Api.inference], providers, provider_data, models=[ @@ -140,4 +141,4 @@ async def memory_stack(embedding_model, request): ], ) - return test_stack.impls[Api.memory], test_stack.impls[Api.memory_banks] + return test_stack.impls[Api.vector_io], test_stack.impls[Api.vector_dbs] diff --git a/llama_stack/providers/tests/memory/fixtures/dummy.pdf b/llama_stack/providers/tests/vector_io/fixtures/dummy.pdf similarity index 100% rename from llama_stack/providers/tests/memory/fixtures/dummy.pdf rename to llama_stack/providers/tests/vector_io/fixtures/dummy.pdf diff --git a/llama_stack/providers/tests/vector_io/test_vector_io.py b/llama_stack/providers/tests/vector_io/test_vector_io.py new file mode 100644 index 000000000..521131f63 --- /dev/null +++ b/llama_stack/providers/tests/vector_io/test_vector_io.py @@ -0,0 +1,199 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +import uuid + +import pytest + +from llama_stack.apis.tools import RAGDocument + +from llama_stack.apis.vector_dbs import ListVectorDBsResponse, VectorDB +from llama_stack.apis.vector_io import QueryChunksResponse + +from llama_stack.providers.utils.memory.vector_store import make_overlapped_chunks + +# How to run this test: +# +# pytest llama_stack/providers/tests/memory/test_memory.py +# -m "sentence_transformers" --env EMBEDDING_DIMENSION=384 +# -v -s --tb=short --disable-warnings + + +@pytest.fixture(scope="session") +def sample_chunks(): + docs = [ + RAGDocument( + document_id="doc1", + content="Python is a high-level programming language.", + metadata={"category": "programming", "difficulty": "beginner"}, + ), + RAGDocument( + document_id="doc2", + content="Machine learning is a subset of artificial intelligence.", + metadata={"category": "AI", "difficulty": "advanced"}, + ), + RAGDocument( + document_id="doc3", + content="Data structures are fundamental to computer science.", + metadata={"category": "computer science", "difficulty": "intermediate"}, + ), + RAGDocument( + document_id="doc4", + content="Neural networks are inspired by biological neural networks.", + metadata={"category": "AI", "difficulty": "advanced"}, + ), + ] + chunks = [] + for doc in docs: + chunks.extend( + make_overlapped_chunks( + doc.document_id, doc.content, window_len=512, overlap_len=64 + ) + ) + return chunks + + +async def register_vector_db(vector_dbs_impl: VectorDB, embedding_model: str): + vector_db_id = f"test_vector_db_{uuid.uuid4().hex}" + return await vector_dbs_impl.register_vector_db( + vector_db_id=vector_db_id, + embedding_model=embedding_model, + embedding_dimension=384, + ) + + +class TestVectorIO: + @pytest.mark.asyncio + async def test_banks_list(self, vector_io_stack, embedding_model): + _, vector_dbs_impl = vector_io_stack + + # Register a test bank + registered_vector_db = await register_vector_db( + vector_dbs_impl, embedding_model + ) + + try: + # Verify our bank shows up in list + response = await vector_dbs_impl.list_vector_dbs() + assert isinstance(response, ListVectorDBsResponse) + assert any( + vector_db.vector_db_id == registered_vector_db.vector_db_id + for vector_db in response.data + ) + finally: + # Clean up + await vector_dbs_impl.unregister_vector_db( + registered_vector_db.vector_db_id + ) + + # Verify our bank was removed + response = await vector_dbs_impl.list_vector_dbs() + assert isinstance(response, ListVectorDBsResponse) + assert all( + vector_db.vector_db_id != registered_vector_db.vector_db_id + for vector_db in response.data + ) + + @pytest.mark.asyncio + async def test_banks_register(self, vector_io_stack, embedding_model): + _, vector_dbs_impl = vector_io_stack + + vector_db_id = f"test_vector_db_{uuid.uuid4().hex}" + + try: + # Register initial bank + await vector_dbs_impl.register_vector_db( + vector_db_id=vector_db_id, + embedding_model=embedding_model, + embedding_dimension=384, + ) + + # Verify our bank exists + response = await vector_dbs_impl.list_vector_dbs() + assert isinstance(response, ListVectorDBsResponse) + assert any( + vector_db.vector_db_id == vector_db_id for vector_db in response.data + ) + + # Try registering same bank again + await vector_dbs_impl.register_vector_db( + vector_db_id=vector_db_id, + embedding_model=embedding_model, + embedding_dimension=384, + ) + + # Verify still only one instance of our bank + response = await vector_dbs_impl.list_vector_dbs() + assert isinstance(response, ListVectorDBsResponse) + assert ( + len( + [ + vector_db + for vector_db in response.data + if vector_db.vector_db_id == vector_db_id + ] + ) + == 1 + ) + finally: + # Clean up + await vector_dbs_impl.unregister_vector_db(vector_db_id) + + @pytest.mark.asyncio + async def test_query_documents( + self, vector_io_stack, embedding_model, sample_chunks + ): + vector_io_impl, vector_dbs_impl = vector_io_stack + + with pytest.raises(ValueError): + await vector_io_impl.insert_chunks("test_vector_db", sample_chunks) + + registered_db = await register_vector_db(vector_dbs_impl, embedding_model) + await vector_io_impl.insert_chunks(registered_db.vector_db_id, sample_chunks) + + query1 = "programming language" + response1 = await vector_io_impl.query_chunks( + registered_db.vector_db_id, query1 + ) + assert_valid_response(response1) + assert any("Python" in chunk.content for chunk in response1.chunks) + + # Test case 3: Query with semantic similarity + query3 = "AI and brain-inspired computing" + response3 = await vector_io_impl.query_chunks( + registered_db.vector_db_id, query3 + ) + assert_valid_response(response3) + assert any( + "neural networks" in chunk.content.lower() for chunk in response3.chunks + ) + + # Test case 4: Query with limit on number of results + query4 = "computer" + params4 = {"max_chunks": 2} + response4 = await vector_io_impl.query_chunks( + registered_db.vector_db_id, query4, params4 + ) + assert_valid_response(response4) + assert len(response4.chunks) <= 2 + + # Test case 5: Query with threshold on similarity score + query5 = "quantum computing" # Not directly related to any document + params5 = {"score_threshold": 0.01} + response5 = await vector_io_impl.query_chunks( + registered_db.vector_db_id, query5, params5 + ) + assert_valid_response(response5) + print("The scores are:", response5.scores) + assert all(score >= 0.01 for score in response5.scores) + + +def assert_valid_response(response: QueryChunksResponse): + assert len(response.chunks) > 0 + assert len(response.scores) > 0 + assert len(response.chunks) == len(response.scores) + for chunk in response.chunks: + assert isinstance(chunk.content, str) diff --git a/llama_stack/providers/tests/memory/test_vector_store.py b/llama_stack/providers/tests/vector_io/test_vector_store.py similarity index 87% rename from llama_stack/providers/tests/memory/test_vector_store.py rename to llama_stack/providers/tests/vector_io/test_vector_store.py index 1ad7abf0c..2a41a8982 100644 --- a/llama_stack/providers/tests/memory/test_vector_store.py +++ b/llama_stack/providers/tests/vector_io/test_vector_store.py @@ -11,8 +11,9 @@ from pathlib import Path import pytest -from llama_stack.apis.memory.memory import MemoryBankDocument, URL -from llama_stack.providers.utils.memory.vector_store import content_from_doc +from llama_stack.apis.tools import RAGDocument + +from llama_stack.providers.utils.memory.vector_store import content_from_doc, URL DUMMY_PDF_PATH = Path(os.path.abspath(__file__)).parent / "fixtures" / "dummy.pdf" @@ -38,33 +39,33 @@ class TestVectorStore: @pytest.mark.asyncio async def test_returns_content_from_pdf_data_uri(self): data_uri = data_url_from_file(DUMMY_PDF_PATH) - doc = MemoryBankDocument( + doc = RAGDocument( document_id="dummy", content=data_uri, mime_type="application/pdf", metadata={}, ) content = await content_from_doc(doc) - assert content == "Dummy PDF file" + assert content == "Dumm y PDF file" @pytest.mark.asyncio async def test_downloads_pdf_and_returns_content(self): # Using GitHub to host the PDF file url = "https://raw.githubusercontent.com/meta-llama/llama-stack/da035d69cfca915318eaf485770a467ca3c2a238/llama_stack/providers/tests/memory/fixtures/dummy.pdf" - doc = MemoryBankDocument( + doc = RAGDocument( document_id="dummy", content=url, mime_type="application/pdf", metadata={}, ) content = await content_from_doc(doc) - assert content == "Dummy PDF file" + assert content == "Dumm y PDF file" @pytest.mark.asyncio async def test_downloads_pdf_and_returns_content_with_url_object(self): # Using GitHub to host the PDF file url = "https://raw.githubusercontent.com/meta-llama/llama-stack/da035d69cfca915318eaf485770a467ca3c2a238/llama_stack/providers/tests/memory/fixtures/dummy.pdf" - doc = MemoryBankDocument( + doc = RAGDocument( document_id="dummy", content=URL( uri=url, @@ -73,4 +74,4 @@ class TestVectorStore: metadata={}, ) content = await content_from_doc(doc) - assert content == "Dummy PDF file" + assert content == "Dumm y PDF file" diff --git a/llama_stack/providers/utils/common/data_schema_validator.py b/llama_stack/providers/utils/common/data_schema_validator.py index af58a4592..55f1078a4 100644 --- a/llama_stack/providers/utils/common/data_schema_validator.py +++ b/llama_stack/providers/utils/common/data_schema_validator.py @@ -23,6 +23,7 @@ class ColumnName(Enum): completion_input = "completion_input" generated_answer = "generated_answer" context = "context" + dialog = "dialog" VALID_SCHEMAS_FOR_SCORING = [ diff --git a/llama_stack/providers/utils/inference/openai_compat.py b/llama_stack/providers/utils/inference/openai_compat.py index 4c46954cf..6c93f49c0 100644 --- a/llama_stack/providers/utils/inference/openai_compat.py +++ b/llama_stack/providers/utils/inference/openai_compat.py @@ -4,11 +4,17 @@ # This source code is licensed under the terms described in the LICENSE file in # the root directory of this source tree. -from typing import AsyncGenerator, List, Optional +from typing import AsyncGenerator, Dict, List, Optional + +from llama_models.datatypes import ( + GreedySamplingStrategy, + SamplingParams, + TopKSamplingStrategy, + TopPSamplingStrategy, +) from llama_models.llama3.api.chat_format import ChatFormat - -from llama_models.llama3.api.datatypes import SamplingParams, StopReason +from llama_models.llama3.api.datatypes import StopReason from pydantic import BaseModel from llama_stack.apis.common.content_types import ( @@ -28,6 +34,7 @@ from llama_stack.apis.inference import ( CompletionResponse, CompletionResponseStreamChunk, Message, + TokenLogProbs, ) from llama_stack.providers.utils.inference.prompt_adapter import ( @@ -39,22 +46,48 @@ class OpenAICompatCompletionChoiceDelta(BaseModel): content: str +class OpenAICompatLogprobs(BaseModel): + text_offset: Optional[List[int]] = None + + token_logprobs: Optional[List[float]] = None + + tokens: Optional[List[str]] = None + + top_logprobs: Optional[List[Dict[str, float]]] = None + + class OpenAICompatCompletionChoice(BaseModel): finish_reason: Optional[str] = None text: Optional[str] = None delta: Optional[OpenAICompatCompletionChoiceDelta] = None + logprobs: Optional[OpenAICompatLogprobs] = None class OpenAICompatCompletionResponse(BaseModel): choices: List[OpenAICompatCompletionChoice] +def get_sampling_strategy_options(params: SamplingParams) -> dict: + options = {} + if isinstance(params.strategy, GreedySamplingStrategy): + options["temperature"] = 0.0 + elif isinstance(params.strategy, TopPSamplingStrategy): + options["temperature"] = params.strategy.temperature + options["top_p"] = params.strategy.top_p + elif isinstance(params.strategy, TopKSamplingStrategy): + options["top_k"] = params.strategy.top_k + else: + raise ValueError(f"Unsupported sampling strategy: {params.strategy}") + + return options + + def get_sampling_options(params: SamplingParams) -> dict: options = {} if params: - for attr in {"temperature", "top_p", "top_k", "max_tokens"}: - if getattr(params, attr): - options[attr] = getattr(params, attr) + options.update(get_sampling_strategy_options(params)) + if params.max_tokens: + options["max_tokens"] = params.max_tokens if params.repetition_penalty is not None and params.repetition_penalty != 1.0: options["repeat_penalty"] = params.repetition_penalty @@ -83,6 +116,14 @@ def get_stop_reason(finish_reason: str) -> StopReason: return StopReason.out_of_tokens +def convert_openai_completion_logprobs( + logprobs: Optional[OpenAICompatLogprobs], +) -> Optional[List[TokenLogProbs]]: + if not logprobs: + return None + return [TokenLogProbs(logprobs_by_token=x) for x in logprobs.top_logprobs] + + def process_completion_response( response: OpenAICompatCompletionResponse, formatter: ChatFormat ) -> CompletionResponse: @@ -92,16 +133,19 @@ def process_completion_response( return CompletionResponse( stop_reason=StopReason.end_of_turn, content=choice.text[: -len("<|eot_id|>")], + logprobs=convert_openai_completion_logprobs(choice.logprobs), ) # drop suffix if present and return stop reason as end of message if choice.text.endswith("<|eom_id|>"): return CompletionResponse( stop_reason=StopReason.end_of_message, content=choice.text[: -len("<|eom_id|>")], + logprobs=convert_openai_completion_logprobs(choice.logprobs), ) return CompletionResponse( stop_reason=get_stop_reason(choice.finish_reason), content=choice.text, + logprobs=convert_openai_completion_logprobs(choice.logprobs), ) @@ -144,6 +188,7 @@ async def process_completion_stream_response( yield CompletionResponseStreamChunk( delta=text, stop_reason=stop_reason, + logprobs=convert_openai_completion_logprobs(choice.logprobs), ) if finish_reason: if finish_reason in ["stop", "eos", "eos_token"]: @@ -195,7 +240,7 @@ async def process_chat_completion_stream_response( event=ChatCompletionResponseEvent( event_type=ChatCompletionResponseEventType.progress, delta=ToolCallDelta( - content="", + tool_call="", parse_status=ToolCallParseStatus.started, ), ) @@ -215,7 +260,7 @@ async def process_chat_completion_stream_response( if ipython: buffer += text delta = ToolCallDelta( - content=text, + tool_call=text, parse_status=ToolCallParseStatus.in_progress, ) @@ -244,7 +289,7 @@ async def process_chat_completion_stream_response( event=ChatCompletionResponseEvent( event_type=ChatCompletionResponseEventType.progress, delta=ToolCallDelta( - content="", + tool_call="", parse_status=ToolCallParseStatus.failed, ), stop_reason=stop_reason, @@ -256,7 +301,7 @@ async def process_chat_completion_stream_response( event=ChatCompletionResponseEvent( event_type=ChatCompletionResponseEventType.progress, delta=ToolCallDelta( - content=tool_call, + tool_call=tool_call, parse_status=ToolCallParseStatus.succeeded, ), stop_reason=stop_reason, diff --git a/llama_stack/providers/utils/inference/prompt_adapter.py b/llama_stack/providers/utils/inference/prompt_adapter.py index de4918f5c..f5298d844 100644 --- a/llama_stack/providers/utils/inference/prompt_adapter.py +++ b/llama_stack/providers/utils/inference/prompt_adapter.py @@ -113,28 +113,29 @@ async def interleaved_content_convert_to_raw( elif isinstance(c, TextContentItem): return RawTextItem(text=c.text) elif isinstance(c, ImageContentItem): - if c.url: + image = c.image + if image.url: # Load image bytes from URL - if c.url.uri.startswith("data"): - match = re.match(r"data:image/(\w+);base64,(.+)", c.url.uri) + if image.url.uri.startswith("data"): + match = re.match(r"data:image/(\w+);base64,(.+)", image.url.uri) if not match: raise ValueError( - f"Invalid data URL format, {c.url.uri[:40]}..." + f"Invalid data URL format, {image.url.uri[:40]}..." ) _, image_data = match.groups() data = base64.b64decode(image_data) - elif c.url.uri.startswith("file://"): - path = c.url.uri[len("file://") :] + elif image.url.uri.startswith("file://"): + path = image.url.uri[len("file://") :] with open(path, "rb") as f: data = f.read() # type: ignore - elif c.url.uri.startswith("http"): + elif image.url.uri.startswith("http"): async with httpx.AsyncClient() as client: - response = await client.get(c.url.uri) + response = await client.get(image.url.uri) data = response.content else: raise ValueError("Unsupported URL type") - elif c.data: - data = c.data + elif image.data: + data = image.data else: raise ValueError("No data or URL provided") @@ -170,26 +171,29 @@ def request_has_media(request: Union[ChatCompletionRequest, CompletionRequest]): async def localize_image_content(media: ImageContentItem) -> Tuple[bytes, str]: - if media.url and media.url.uri.startswith("http"): + image = media.image + if image.url and image.url.uri.startswith("http"): async with httpx.AsyncClient() as client: - r = await client.get(media.url.uri) + r = await client.get(image.url.uri) content = r.content content_type = r.headers.get("content-type") if content_type: format = content_type.split("/")[-1] else: format = "png" + return content, format else: - image = PIL_Image.open(io.BytesIO(media.data)) - return media.data, image.format + pil_image = PIL_Image.open(io.BytesIO(image.data)) + return image.data, pil_image.format async def convert_image_content_to_url( media: ImageContentItem, download: bool = False, include_format: bool = True ) -> str: - if media.url and not download: - return media.url.uri + image = media.image + if image.url and (not download or image.url.uri.startswith("data")): + return image.url.uri content, format = await localize_image_content(media) if include_format: @@ -227,9 +231,11 @@ async def completion_request_to_prompt_model_input_info( def augment_content_with_response_format_prompt(response_format, content): if fmt_prompt := response_format_prompt(response_format): if isinstance(content, list): - return content + [fmt_prompt] + return content + [TextContentItem(text=fmt_prompt)] + elif isinstance(content, str): + return [TextContentItem(text=content), TextContentItem(text=fmt_prompt)] else: - return [content, fmt_prompt] + return [content, TextContentItem(text=fmt_prompt)] return content diff --git a/llama_stack/providers/utils/memory/vector_store.py b/llama_stack/providers/utils/memory/vector_store.py index c97633558..82c0c9c07 100644 --- a/llama_stack/providers/utils/memory/vector_store.py +++ b/llama_stack/providers/utils/memory/vector_store.py @@ -18,6 +18,7 @@ import numpy as np from llama_models.llama3.api.tokenizer import Tokenizer from numpy.typing import NDArray + from pypdf import PdfReader from llama_stack.apis.common.content_types import ( @@ -25,8 +26,9 @@ from llama_stack.apis.common.content_types import ( TextContentItem, URL, ) -from llama_stack.apis.memory import Chunk, MemoryBankDocument, QueryDocumentsResponse -from llama_stack.apis.memory_banks import VectorMemoryBank +from llama_stack.apis.tools import RAGDocument +from llama_stack.apis.vector_dbs import VectorDB +from llama_stack.apis.vector_io import Chunk, QueryChunksResponse from llama_stack.providers.datatypes import Api from llama_stack.providers.utils.inference.prompt_adapter import ( interleaved_content_as_str, @@ -112,7 +114,7 @@ def concat_interleaved_content(content: List[InterleavedContent]) -> Interleaved return ret -async def content_from_doc(doc: MemoryBankDocument) -> str: +async def content_from_doc(doc: RAGDocument) -> str: if isinstance(doc.content, URL): if doc.content.uri.startswith("data:"): return content_from_data(doc.content.uri) @@ -151,7 +153,13 @@ def make_overlapped_chunks( chunk = tokenizer.decode(toks) # chunk is a string chunks.append( - Chunk(content=chunk, token_count=len(toks), document_id=document_id) + Chunk( + content=chunk, + metadata={ + "token_count": len(toks), + "document_id": document_id, + }, + ) ) return chunks @@ -165,7 +173,7 @@ class EmbeddingIndex(ABC): @abstractmethod async def query( self, embedding: NDArray, k: int, score_threshold: float - ) -> QueryDocumentsResponse: + ) -> QueryChunksResponse: raise NotImplementedError() @abstractmethod @@ -174,56 +182,35 @@ class EmbeddingIndex(ABC): @dataclass -class BankWithIndex: - bank: VectorMemoryBank +class VectorDBWithIndex: + vector_db: VectorDB index: EmbeddingIndex inference_api: Api.inference - async def insert_documents( + async def insert_chunks( self, - documents: List[MemoryBankDocument], + chunks: List[Chunk], ) -> None: - for doc in documents: - content = await content_from_doc(doc) - chunks = make_overlapped_chunks( - doc.document_id, - content, - self.bank.chunk_size_in_tokens, - self.bank.overlap_size_in_tokens - or (self.bank.chunk_size_in_tokens // 4), - ) - if not chunks: - continue - embeddings_response = await self.inference_api.embeddings( - self.bank.embedding_model, [x.content for x in chunks] - ) - embeddings = np.array(embeddings_response.embeddings) + embeddings_response = await self.inference_api.embeddings( + self.vector_db.embedding_model, [x.content for x in chunks] + ) + embeddings = np.array(embeddings_response.embeddings) - await self.index.add_chunks(chunks, embeddings) + await self.index.add_chunks(chunks, embeddings) - async def query_documents( + async def query_chunks( self, query: InterleavedContent, params: Optional[Dict[str, Any]] = None, - ) -> QueryDocumentsResponse: + ) -> QueryChunksResponse: if params is None: params = {} k = params.get("max_chunks", 3) score_threshold = params.get("score_threshold", 0.0) - def _process(c) -> str: - if isinstance(c, str): - return c - else: - return "" - - if isinstance(query, list): - query_str = " ".join([_process(c) for c in query]) - else: - query_str = _process(query) - + query_str = interleaved_content_as_str(query) embeddings_response = await self.inference_api.embeddings( - self.bank.embedding_model, [query_str] + self.vector_db.embedding_model, [query_str] ) query_vector = np.array(embeddings_response.embeddings[0], dtype=np.float32) return await self.index.query(query_vector, k, score_threshold) diff --git a/llama_stack/providers/utils/telemetry/dataset_mixin.py b/llama_stack/providers/utils/telemetry/dataset_mixin.py index bf5e79c3d..a2bfdcb87 100644 --- a/llama_stack/providers/utils/telemetry/dataset_mixin.py +++ b/llama_stack/providers/utils/telemetry/dataset_mixin.py @@ -7,7 +7,7 @@ from typing import List, Optional from llama_stack.apis.datasetio import DatasetIO -from llama_stack.apis.telemetry import QueryCondition, Span +from llama_stack.apis.telemetry import QueryCondition, QuerySpansResponse, Span class TelemetryDatasetMixin: @@ -22,6 +22,9 @@ class TelemetryDatasetMixin: dataset_id: str, max_depth: Optional[int] = None, ) -> None: + if self.datasetio_api is None: + raise RuntimeError("DatasetIO API not available") + spans = await self.query_spans( attribute_filters=attribute_filters, attributes_to_return=attributes_to_save, @@ -48,18 +51,18 @@ class TelemetryDatasetMixin: attribute_filters: List[QueryCondition], attributes_to_return: List[str], max_depth: Optional[int] = None, - ) -> List[Span]: + ) -> QuerySpansResponse: traces = await self.query_traces(attribute_filters=attribute_filters) spans = [] - for trace in traces: - spans_by_id = await self.get_span_tree( + for trace in traces.data: + spans_by_id_resp = await self.get_span_tree( span_id=trace.root_span_id, attributes_to_return=attributes_to_return, max_depth=max_depth, ) - for span in spans_by_id.values(): + for span in spans_by_id_resp.data.values(): if span.attributes and all( attr in span.attributes and span.attributes[attr] is not None for attr in attributes_to_return @@ -76,4 +79,4 @@ class TelemetryDatasetMixin: ) ) - return spans + return QuerySpansResponse(data=spans) diff --git a/llama_stack/providers/utils/telemetry/sqlite_trace_store.py b/llama_stack/providers/utils/telemetry/sqlite_trace_store.py index b0c3f7868..a2821da43 100644 --- a/llama_stack/providers/utils/telemetry/sqlite_trace_store.py +++ b/llama_stack/providers/utils/telemetry/sqlite_trace_store.py @@ -10,7 +10,7 @@ from typing import Dict, List, Optional, Protocol import aiosqlite -from llama_stack.apis.telemetry import QueryCondition, SpanWithStatus, Trace +from llama_stack.apis.telemetry import QueryCondition, Span, SpanWithStatus, Trace class TraceStore(Protocol): @@ -167,3 +167,23 @@ class SQLiteTraceStore(TraceStore): spans_by_id[span.span_id] = span return spans_by_id + + async def get_trace(self, trace_id: str) -> Trace: + query = "SELECT * FROM traces WHERE trace_id = ?" + async with aiosqlite.connect(self.conn_string) as conn: + conn.row_factory = aiosqlite.Row + async with conn.execute(query, (trace_id,)) as cursor: + row = await cursor.fetchone() + if row is None: + raise ValueError(f"Trace {trace_id} not found") + return Trace(**row) + + async def get_span(self, trace_id: str, span_id: str) -> Span: + query = "SELECT * FROM spans WHERE trace_id = ? AND span_id = ?" + async with aiosqlite.connect(self.conn_string) as conn: + conn.row_factory = aiosqlite.Row + async with conn.execute(query, (trace_id, span_id)) as cursor: + row = await cursor.fetchone() + if row is None: + raise ValueError(f"Span {span_id} not found") + return Span(**row) diff --git a/llama_stack/providers/utils/telemetry/tracing.py b/llama_stack/providers/utils/telemetry/tracing.py index f304d58f6..d84024941 100644 --- a/llama_stack/providers/utils/telemetry/tracing.py +++ b/llama_stack/providers/utils/telemetry/tracing.py @@ -127,7 +127,8 @@ class TraceContext: def setup_logger(api: Telemetry, level: int = logging.INFO): global BACKGROUND_LOGGER - BACKGROUND_LOGGER = BackgroundLogger(api) + if BACKGROUND_LOGGER is None: + BACKGROUND_LOGGER = BackgroundLogger(api) logger = logging.getLogger() logger.setLevel(level) logger.addHandler(TelemetryHandler()) diff --git a/llama_stack/scripts/test_rag_via_curl.py b/llama_stack/scripts/test_rag_via_curl.py new file mode 100644 index 000000000..28d6fb601 --- /dev/null +++ b/llama_stack/scripts/test_rag_via_curl.py @@ -0,0 +1,105 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +import json +from typing import List + +import pytest +import requests +from pydantic import TypeAdapter + +from llama_stack.apis.tools import ( + DefaultRAGQueryGeneratorConfig, + RAGDocument, + RAGQueryConfig, + RAGQueryResult, +) +from llama_stack.apis.vector_dbs import VectorDB +from llama_stack.providers.utils.memory.vector_store import interleaved_content_as_str + + +class TestRAGToolEndpoints: + @pytest.fixture + def base_url(self) -> str: + return "http://localhost:8321/v1" # Adjust port if needed + + @pytest.fixture + def sample_documents(self) -> List[RAGDocument]: + return [ + RAGDocument( + document_id="doc1", + content="Python is a high-level programming language.", + metadata={"category": "programming", "difficulty": "beginner"}, + ), + RAGDocument( + document_id="doc2", + content="Machine learning is a subset of artificial intelligence.", + metadata={"category": "AI", "difficulty": "advanced"}, + ), + RAGDocument( + document_id="doc3", + content="Data structures are fundamental to computer science.", + metadata={"category": "computer science", "difficulty": "intermediate"}, + ), + ] + + @pytest.mark.asyncio + async def test_rag_workflow( + self, base_url: str, sample_documents: List[RAGDocument] + ): + vector_db_payload = { + "vector_db_id": "test_vector_db", + "embedding_model": "all-MiniLM-L6-v2", + "embedding_dimension": 384, + } + + response = requests.post(f"{base_url}/vector-dbs", json=vector_db_payload) + assert response.status_code == 200 + vector_db = VectorDB(**response.json()) + + insert_payload = { + "documents": [ + json.loads(doc.model_dump_json()) for doc in sample_documents + ], + "vector_db_id": vector_db.identifier, + "chunk_size_in_tokens": 512, + } + + response = requests.post( + f"{base_url}/tool-runtime/rag-tool/insert-documents", + json=insert_payload, + ) + assert response.status_code == 200 + + query = "What is Python?" + query_config = RAGQueryConfig( + query_generator_config=DefaultRAGQueryGeneratorConfig(), + max_tokens_in_context=4096, + max_chunks=2, + ) + + query_payload = { + "content": query, + "query_config": json.loads(query_config.model_dump_json()), + "vector_db_ids": [vector_db.identifier], + } + + response = requests.post( + f"{base_url}/tool-runtime/rag-tool/query-context", + json=query_payload, + ) + assert response.status_code == 200 + result = response.json() + result = TypeAdapter(RAGQueryResult).validate_python(result) + + content_str = interleaved_content_as_str(result.content) + print(f"content: {content_str}") + assert len(content_str) > 0 + assert "Python" in content_str + + # Clean up: Delete the vector DB + response = requests.delete(f"{base_url}/vector-dbs/{vector_db.identifier}") + assert response.status_code == 200 diff --git a/llama_stack/templates/bedrock/bedrock.py b/llama_stack/templates/bedrock/bedrock.py index c80625cf6..6b83e9536 100644 --- a/llama_stack/templates/bedrock/bedrock.py +++ b/llama_stack/templates/bedrock/bedrock.py @@ -10,7 +10,7 @@ from llama_models.sku_list import all_registered_models from llama_stack.apis.models import ModelInput from llama_stack.distribution.datatypes import Provider, ToolGroupInput -from llama_stack.providers.inline.memory.faiss.config import FaissImplConfig +from llama_stack.providers.inline.vector_io.faiss.config import FaissImplConfig from llama_stack.providers.remote.inference.bedrock.bedrock import MODEL_ALIASES from llama_stack.templates.template import DistributionTemplate, RunConfigSettings @@ -18,7 +18,7 @@ from llama_stack.templates.template import DistributionTemplate, RunConfigSettin def get_distribution_template() -> DistributionTemplate: providers = { "inference": ["remote::bedrock"], - "memory": ["inline::faiss", "remote::chromadb", "remote::pgvector"], + "vector_io": ["inline::faiss", "remote::chromadb", "remote::pgvector"], "safety": ["remote::bedrock"], "agents": ["inline::meta-reference"], "telemetry": ["inline::meta-reference"], @@ -29,11 +29,12 @@ def get_distribution_template() -> DistributionTemplate: "remote::brave-search", "remote::tavily-search", "inline::code-interpreter", - "inline::memory-runtime", + "inline::rag-runtime", + "remote::model-context-protocol", ], } name = "bedrock" - memory_provider = Provider( + vector_io_provider = Provider( provider_id="faiss", provider_type="inline::faiss", config=FaissImplConfig.sample_run_config(f"distributions/{name}"), @@ -57,8 +58,8 @@ def get_distribution_template() -> DistributionTemplate: provider_id="tavily-search", ), ToolGroupInput( - toolgroup_id="builtin::memory", - provider_id="memory-runtime", + toolgroup_id="builtin::rag", + provider_id="rag-runtime", ), ToolGroupInput( toolgroup_id="builtin::code_interpreter", @@ -70,14 +71,14 @@ def get_distribution_template() -> DistributionTemplate: name=name, distro_type="self_hosted", description="Use AWS Bedrock for running LLM inference and safety", - docker_image=None, + container_image=None, template_path=Path(__file__).parent / "doc_template.md", providers=providers, default_models=default_models, run_configs={ "run.yaml": RunConfigSettings( provider_overrides={ - "memory": [memory_provider], + "vector_io": [vector_io_provider], }, default_models=default_models, default_tool_groups=default_tool_groups, diff --git a/llama_stack/templates/bedrock/build.yaml b/llama_stack/templates/bedrock/build.yaml index a68a8f6fc..6c07b0478 100644 --- a/llama_stack/templates/bedrock/build.yaml +++ b/llama_stack/templates/bedrock/build.yaml @@ -1,11 +1,10 @@ version: '2' -name: bedrock distribution_spec: description: Use AWS Bedrock for running LLM inference and safety providers: inference: - remote::bedrock - memory: + vector_io: - inline::faiss - remote::chromadb - remote::pgvector @@ -28,5 +27,6 @@ distribution_spec: - remote::brave-search - remote::tavily-search - inline::code-interpreter - - inline::memory-runtime + - inline::rag-runtime + - remote::model-context-protocol image_type: conda diff --git a/llama_stack/templates/bedrock/run.yaml b/llama_stack/templates/bedrock/run.yaml index 1d0721773..39408c1bd 100644 --- a/llama_stack/templates/bedrock/run.yaml +++ b/llama_stack/templates/bedrock/run.yaml @@ -1,22 +1,21 @@ version: '2' image_name: bedrock -conda_env: bedrock apis: - agents - datasetio - eval - inference -- memory - safety - scoring - telemetry - tool_runtime +- vector_io providers: inference: - provider_id: bedrock provider_type: remote::bedrock config: {} - memory: + vector_io: - provider_id: faiss provider_type: inline::faiss config: @@ -79,8 +78,11 @@ providers: - provider_id: code-interpreter provider_type: inline::code-interpreter config: {} - - provider_id: memory-runtime - provider_type: inline::memory-runtime + - provider_id: rag-runtime + provider_type: inline::rag-runtime + config: {} + - provider_id: model-context-protocol + provider_type: remote::model-context-protocol config: {} metadata_store: type: sqlite @@ -102,14 +104,14 @@ models: provider_model_id: meta.llama3-1-405b-instruct-v1:0 model_type: llm shields: [] -memory_banks: [] +vector_dbs: [] datasets: [] scoring_fns: [] eval_tasks: [] tool_groups: - toolgroup_id: builtin::websearch provider_id: tavily-search -- toolgroup_id: builtin::memory - provider_id: memory-runtime +- toolgroup_id: builtin::rag + provider_id: rag-runtime - toolgroup_id: builtin::code_interpreter provider_id: code-interpreter diff --git a/llama_stack/templates/cerebras/build.yaml b/llama_stack/templates/cerebras/build.yaml index 307e0303a..9d5ab1a52 100644 --- a/llama_stack/templates/cerebras/build.yaml +++ b/llama_stack/templates/cerebras/build.yaml @@ -1,5 +1,4 @@ version: '2' -name: cerebras distribution_spec: description: Use Cerebras for running LLM inference providers: @@ -7,15 +6,26 @@ distribution_spec: - remote::cerebras safety: - inline::llama-guard - memory: - - inline::meta-reference + vector_io: + - inline::faiss + - remote::chromadb + - remote::pgvector agents: - inline::meta-reference + eval: + - inline::meta-reference + datasetio: + - remote::huggingface + - inline::localfs + scoring: + - inline::basic + - inline::llm-as-judge + - inline::braintrust telemetry: - inline::meta-reference tool_runtime: - remote::brave-search - remote::tavily-search - inline::code-interpreter - - inline::memory-runtime + - inline::rag-runtime image_type: conda diff --git a/llama_stack/templates/cerebras/cerebras.py b/llama_stack/templates/cerebras/cerebras.py index b51617f35..50a878645 100644 --- a/llama_stack/templates/cerebras/cerebras.py +++ b/llama_stack/templates/cerebras/cerebras.py @@ -9,15 +9,11 @@ from pathlib import Path from llama_models.sku_list import all_registered_models from llama_stack.apis.models.models import ModelType -from llama_stack.distribution.datatypes import ( - ModelInput, - Provider, - ShieldInput, - ToolGroupInput, -) +from llama_stack.distribution.datatypes import ModelInput, Provider, ToolGroupInput from llama_stack.providers.inline.inference.sentence_transformers import ( SentenceTransformersInferenceConfig, ) +from llama_stack.providers.inline.vector_io.faiss.config import FaissImplConfig from llama_stack.providers.remote.inference.cerebras import CerebrasImplConfig from llama_stack.providers.remote.inference.cerebras.cerebras import model_aliases from llama_stack.templates.template import DistributionTemplate, RunConfigSettings @@ -27,17 +23,21 @@ def get_distribution_template() -> DistributionTemplate: providers = { "inference": ["remote::cerebras"], "safety": ["inline::llama-guard"], - "memory": ["inline::meta-reference"], + "vector_io": ["inline::faiss", "remote::chromadb", "remote::pgvector"], "agents": ["inline::meta-reference"], + "eval": ["inline::meta-reference"], + "datasetio": ["remote::huggingface", "inline::localfs"], + "scoring": ["inline::basic", "inline::llm-as-judge", "inline::braintrust"], "telemetry": ["inline::meta-reference"], "tool_runtime": [ "remote::brave-search", "remote::tavily-search", "inline::code-interpreter", - "inline::memory-runtime", + "inline::rag-runtime", ], } + name = "cerebras" inference_provider = Provider( provider_id="cerebras", provider_type="remote::cerebras", @@ -68,14 +68,19 @@ def get_distribution_template() -> DistributionTemplate: "embedding_dimension": 384, }, ) + vector_io_provider = Provider( + provider_id="faiss", + provider_type="inline::faiss", + config=FaissImplConfig.sample_run_config(f"distributions/{name}"), + ) default_tool_groups = [ ToolGroupInput( toolgroup_id="builtin::websearch", provider_id="tavily-search", ), ToolGroupInput( - toolgroup_id="builtin::memory", - provider_id="memory-runtime", + toolgroup_id="builtin::rag", + provider_id="rag-runtime", ), ToolGroupInput( toolgroup_id="builtin::code_interpreter", @@ -87,7 +92,7 @@ def get_distribution_template() -> DistributionTemplate: name="cerebras", distro_type="self_hosted", description="Use Cerebras for running LLM inference", - docker_image=None, + container_image=None, template_path=Path(__file__).parent / "doc_template.md", providers=providers, default_models=default_models, @@ -95,9 +100,10 @@ def get_distribution_template() -> DistributionTemplate: "run.yaml": RunConfigSettings( provider_overrides={ "inference": [inference_provider, embedding_provider], + "vector_io": [vector_io_provider], }, default_models=default_models + [embedding_model], - default_shields=[ShieldInput(shield_id="meta-llama/Llama-Guard-3-8B")], + default_shields=[], default_tool_groups=default_tool_groups, ), }, diff --git a/llama_stack/templates/cerebras/report.md b/llama_stack/templates/cerebras/report.md new file mode 100644 index 000000000..7c09474b1 --- /dev/null +++ b/llama_stack/templates/cerebras/report.md @@ -0,0 +1,44 @@ +# Report for cerebras distribution + +## Supported Models +| Model Descriptor | cerebras | +|:---|:---| +| meta-llama/Llama-3-8B-Instruct | ❌ | +| meta-llama/Llama-3-70B-Instruct | ❌ | +| meta-llama/Llama-3.1-8B-Instruct | ✅ | +| meta-llama/Llama-3.1-70B-Instruct | ❌ | +| meta-llama/Llama-3.1-405B-Instruct-FP8 | ❌ | +| meta-llama/Llama-3.2-1B-Instruct | ❌ | +| meta-llama/Llama-3.2-3B-Instruct | ❌ | +| meta-llama/Llama-3.2-11B-Vision-Instruct | ❌ | +| meta-llama/Llama-3.2-90B-Vision-Instruct | ❌ | +| meta-llama/Llama-3.3-70B-Instruct | ✅ | +| meta-llama/Llama-Guard-3-11B-Vision | ❌ | +| meta-llama/Llama-Guard-3-1B | ❌ | +| meta-llama/Llama-Guard-3-8B | ❌ | +| meta-llama/Llama-Guard-2-8B | ❌ | + +## Inference +| Model | API | Capability | Test | Status | +|:----- |:-----|:-----|:-----|:-----| +| Llama-3.1-8B-Instruct | /chat_completion | streaming | test_text_chat_completion_streaming | ✅ | +| Llama-3.2-11B-Vision-Instruct | /chat_completion | streaming | test_image_chat_completion_streaming | ❌ | +| Llama-3.2-11B-Vision-Instruct | /chat_completion | non_streaming | test_image_chat_completion_non_streaming | ❌ | +| Llama-3.1-8B-Instruct | /chat_completion | non_streaming | test_text_chat_completion_non_streaming | ✅ | +| Llama-3.1-8B-Instruct | /chat_completion | tool_calling | test_text_chat_completion_with_tool_calling_and_streaming | ✅ | +| Llama-3.1-8B-Instruct | /chat_completion | tool_calling | test_text_chat_completion_with_tool_calling_and_non_streaming | ✅ | +| Llama-3.1-8B-Instruct | /completion | streaming | test_text_completion_streaming | ✅ | +| Llama-3.1-8B-Instruct | /completion | non_streaming | test_text_completion_non_streaming | ✅ | +| Llama-3.1-8B-Instruct | /completion | structured_output | test_text_completion_structured_output | ❌ | + +## Vector IO +| API | Capability | Test | Status | +|:-----|:-----|:-----|:-----| +| /retrieve | | test_vector_db_retrieve | ✅ | + +## Agents +| API | Capability | Test | Status | +|:-----|:-----|:-----|:-----| +| /create_agent_turn | rag | test_rag_agent | ✅ | +| /create_agent_turn | custom_tool | test_custom_tool | ❌ | +| /create_agent_turn | code_execution | test_code_interpreter_for_attachments | ✅ | diff --git a/llama_stack/templates/cerebras/run.yaml b/llama_stack/templates/cerebras/run.yaml index e06b17a50..5a70890a8 100644 --- a/llama_stack/templates/cerebras/run.yaml +++ b/llama_stack/templates/cerebras/run.yaml @@ -1,13 +1,15 @@ version: '2' image_name: cerebras -conda_env: cerebras apis: - agents +- datasetio +- eval - inference -- memory - safety +- scoring - telemetry - tool_runtime +- vector_io providers: inference: - provider_id: cerebras @@ -22,9 +24,9 @@ providers: - provider_id: llama-guard provider_type: inline::llama-guard config: {} - memory: - - provider_id: meta-reference - provider_type: inline::meta-reference + vector_io: + - provider_id: faiss + provider_type: inline::faiss config: kvstore: type: sqlite @@ -38,6 +40,28 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/cerebras}/agents_store.db + eval: + - provider_id: meta-reference + provider_type: inline::meta-reference + config: {} + datasetio: + - provider_id: huggingface + provider_type: remote::huggingface + config: {} + - provider_id: localfs + provider_type: inline::localfs + config: {} + scoring: + - provider_id: basic + provider_type: inline::basic + config: {} + - provider_id: llm-as-judge + provider_type: inline::llm-as-judge + config: {} + - provider_id: braintrust + provider_type: inline::braintrust + config: + openai_api_key: ${env.OPENAI_API_KEY:} telemetry: - provider_id: meta-reference provider_type: inline::meta-reference @@ -59,8 +83,8 @@ providers: - provider_id: code-interpreter provider_type: inline::code-interpreter config: {} - - provider_id: memory-runtime - provider_type: inline::memory-runtime + - provider_id: rag-runtime + provider_type: inline::rag-runtime config: {} metadata_store: type: sqlite @@ -81,16 +105,15 @@ models: model_id: all-MiniLM-L6-v2 provider_id: sentence-transformers model_type: embedding -shields: -- shield_id: meta-llama/Llama-Guard-3-8B -memory_banks: [] +shields: [] +vector_dbs: [] datasets: [] scoring_fns: [] eval_tasks: [] tool_groups: - toolgroup_id: builtin::websearch provider_id: tavily-search -- toolgroup_id: builtin::memory - provider_id: memory-runtime +- toolgroup_id: builtin::rag + provider_id: rag-runtime - toolgroup_id: builtin::code_interpreter provider_id: code-interpreter diff --git a/llama_stack/templates/experimental-post-training/build.yaml b/llama_stack/templates/experimental-post-training/build.yaml index aa7695bca..c146d1b37 100644 --- a/llama_stack/templates/experimental-post-training/build.yaml +++ b/llama_stack/templates/experimental-post-training/build.yaml @@ -2,7 +2,7 @@ version: '2' name: experimental-post-training distribution_spec: description: Experimental template for post training - docker_image: null + container_image: null providers: inference: - inline::meta-reference @@ -10,9 +10,11 @@ distribution_spec: - inline::meta-reference scoring: - inline::basic + - inline::braintrust post_training: - inline::torchtune datasetio: + - inline::localfs - remote::huggingface telemetry: - inline::meta-reference @@ -20,6 +22,8 @@ distribution_spec: - inline::meta-reference safety: - inline::llama-guard - memory: + vector_io: - inline::faiss + tool_runtime: + - remote::brave-search image_type: conda diff --git a/llama_stack/templates/experimental-post-training/run.yaml b/llama_stack/templates/experimental-post-training/run.yaml index a654c375e..75d103c9f 100644 --- a/llama_stack/templates/experimental-post-training/run.yaml +++ b/llama_stack/templates/experimental-post-training/run.yaml @@ -1,17 +1,18 @@ version: '2' image_name: experimental-post-training -docker_image: null +container_image: null conda_env: experimental-post-training apis: - agents - datasetio - eval - inference -- memory +- vector_io - safety - scoring - telemetry - post_training +- tool_runtime providers: inference: - provider_id: meta-reference-inference @@ -28,10 +29,17 @@ providers: - provider_id: basic provider_type: inline::basic config: {} + - provider_id: braintrust + provider_type: inline::braintrust + config: + openai_api_key: ${env.OPENAI_API_KEY:} datasetio: - provider_id: huggingface-0 provider_type: remote::huggingface config: {} + - provider_id: localfs + provider_type: inline::localfs + config: {} telemetry: - provider_id: meta-reference provider_type: inline::meta-reference @@ -52,7 +60,7 @@ providers: - provider_id: llama-guard provider_type: inline::llama-guard config: {} - memory: + vector_io: - provider_id: faiss provider_type: inline::faiss config: @@ -60,6 +68,13 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/meta-reference-gpu}/faiss_store.db + tool_runtime: + - provider_id: brave-search + provider_type: remote::brave-search + config: + api_key: ${env.BRAVE_SEARCH_API_KEY:} + max_results: 3 + metadata_store: namespace: null @@ -67,24 +82,7 @@ metadata_store: db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/meta-reference-gpu}/registry.db models: [] shields: [] -memory_banks: [] -datasets: - - dataset_id: alpaca - provider_id: huggingface-0 - url: - uri: https://huggingface.co/datasets/tatsu-lab/alpaca - metadata: - path: tatsu-lab/alpaca - name: - split: train - dataset_schema: - instruction: - type: string - input: - type: string - output: - type: string - text: - type: string +vector_dbs: [] +datasets: [] scoring_fns: [] eval_tasks: [] diff --git a/llama_stack/templates/fireworks/build.yaml b/llama_stack/templates/fireworks/build.yaml index e76cc86f1..cdd60ec2a 100644 --- a/llama_stack/templates/fireworks/build.yaml +++ b/llama_stack/templates/fireworks/build.yaml @@ -1,11 +1,10 @@ version: '2' -name: fireworks distribution_spec: description: Use Fireworks.AI for running LLM inference providers: inference: - remote::fireworks - memory: + vector_io: - inline::faiss - remote::chromadb - remote::pgvector @@ -28,5 +27,6 @@ distribution_spec: - remote::brave-search - remote::tavily-search - inline::code-interpreter - - inline::memory-runtime + - inline::rag-runtime + - remote::model-context-protocol image_type: conda diff --git a/llama_stack/templates/fireworks/fireworks.py b/llama_stack/templates/fireworks/fireworks.py index 5af4b08cc..546a8b82a 100644 --- a/llama_stack/templates/fireworks/fireworks.py +++ b/llama_stack/templates/fireworks/fireworks.py @@ -18,7 +18,7 @@ from llama_stack.distribution.datatypes import ( from llama_stack.providers.inline.inference.sentence_transformers import ( SentenceTransformersInferenceConfig, ) -from llama_stack.providers.inline.memory.faiss.config import FaissImplConfig +from llama_stack.providers.inline.vector_io.faiss.config import FaissImplConfig from llama_stack.providers.remote.inference.fireworks import FireworksImplConfig from llama_stack.providers.remote.inference.fireworks.fireworks import MODEL_ALIASES from llama_stack.templates.template import DistributionTemplate, RunConfigSettings @@ -27,7 +27,7 @@ from llama_stack.templates.template import DistributionTemplate, RunConfigSettin def get_distribution_template() -> DistributionTemplate: providers = { "inference": ["remote::fireworks"], - "memory": ["inline::faiss", "remote::chromadb", "remote::pgvector"], + "vector_io": ["inline::faiss", "remote::chromadb", "remote::pgvector"], "safety": ["inline::llama-guard"], "agents": ["inline::meta-reference"], "telemetry": ["inline::meta-reference"], @@ -38,7 +38,8 @@ def get_distribution_template() -> DistributionTemplate: "remote::brave-search", "remote::tavily-search", "inline::code-interpreter", - "inline::memory-runtime", + "inline::rag-runtime", + "remote::model-context-protocol", ], } @@ -54,7 +55,7 @@ def get_distribution_template() -> DistributionTemplate: provider_type="inline::sentence-transformers", config=SentenceTransformersInferenceConfig.sample_run_config(), ) - memory_provider = Provider( + vector_io_provider = Provider( provider_id="faiss", provider_type="inline::faiss", config=FaissImplConfig.sample_run_config(f"distributions/{name}"), @@ -71,14 +72,6 @@ def get_distribution_template() -> DistributionTemplate: ) for m in MODEL_ALIASES ] - inference_model = ModelInput( - model_id="${env.INFERENCE_MODEL}", - provider_id="fireworks", - ) - safety_model = ModelInput( - model_id="${env.SAFETY_MODEL}", - provider_id="fireworks", - ) embedding_model = ModelInput( model_id="all-MiniLM-L6-v2", provider_id="sentence-transformers", @@ -93,8 +86,8 @@ def get_distribution_template() -> DistributionTemplate: provider_id="tavily-search", ), ToolGroupInput( - toolgroup_id="builtin::memory", - provider_id="memory-runtime", + toolgroup_id="builtin::rag", + provider_id="rag-runtime", ), ToolGroupInput( toolgroup_id="builtin::code_interpreter", @@ -106,7 +99,7 @@ def get_distribution_template() -> DistributionTemplate: name=name, distro_type="self_hosted", description="Use Fireworks.AI for running LLM inference", - docker_image=None, + container_image=None, template_path=Path(__file__).parent / "doc_template.md", providers=providers, default_models=default_models, @@ -114,7 +107,7 @@ def get_distribution_template() -> DistributionTemplate: "run.yaml": RunConfigSettings( provider_overrides={ "inference": [inference_provider, embedding_provider], - "memory": [memory_provider], + "vector_io": [vector_io_provider], }, default_models=default_models + [embedding_model], default_shields=[ShieldInput(shield_id="meta-llama/Llama-Guard-3-8B")], @@ -126,13 +119,18 @@ def get_distribution_template() -> DistributionTemplate: inference_provider, embedding_provider, ], - "memory": [memory_provider], + "vector_io": [vector_io_provider], "safety": [ Provider( provider_id="llama-guard", provider_type="inline::llama-guard", config={}, ), + Provider( + provider_id="llama-guard-vision", + provider_type="inline::llama-guard", + config={}, + ), Provider( provider_id="code-scanner", provider_type="inline::code-scanner", @@ -141,15 +139,18 @@ def get_distribution_template() -> DistributionTemplate: ], }, default_models=[ - inference_model, - safety_model, + *default_models, embedding_model, ], default_shields=[ ShieldInput( - shield_id="${env.SAFETY_MODEL}", + shield_id="meta-llama/Llama-Guard-3-8B", provider_id="llama-guard", ), + ShieldInput( + shield_id="meta-llama/Llama-Guard-3-11B-Vision", + provider_id="llama-guard-vision", + ), ShieldInput( shield_id="CodeScanner", provider_id="code-scanner", diff --git a/llama_stack/templates/fireworks/remote-hosted-report.md b/llama_stack/templates/fireworks/remote-hosted-report.md new file mode 100644 index 000000000..2f3c882b7 --- /dev/null +++ b/llama_stack/templates/fireworks/remote-hosted-report.md @@ -0,0 +1,45 @@ +# Report for fireworks distribution + +## Supported Models +| Model Descriptor | fireworks | +|:---|:---| +| meta-llama/Llama-3-8B-Instruct | ❌ | +| meta-llama/Llama-3-70B-Instruct | ❌ | +| meta-llama/Llama-3.1-8B-Instruct | ❌ | +| meta-llama/Llama-3.1-70B-Instruct | ❌ | +| meta-llama/Llama-3.1-405B-Instruct-FP8 | ❌ | +| meta-llama/Llama-3.2-1B-Instruct | ❌ | +| meta-llama/Llama-3.2-3B-Instruct | ❌ | +| meta-llama/Llama-3.2-11B-Vision-Instruct | ❌ | +| meta-llama/Llama-3.2-90B-Vision-Instruct | ❌ | +| meta-llama/Llama-3.3-70B-Instruct | ❌ | +| meta-llama/Llama-Guard-3-11B-Vision | ❌ | +| meta-llama/Llama-Guard-3-1B | ❌ | +| meta-llama/Llama-Guard-3-8B | ❌ | +| meta-llama/Llama-Guard-2-8B | ❌ | + +## Inference +| Model | API | Capability | Test | Status | +|:----- |:-----|:-----|:-----|:-----| +| Text | /chat_completion | streaming | test_text_chat_completion_streaming | ❌ | +| Vision | /chat_completion | streaming | test_image_chat_completion_streaming | ❌ | +| Vision | /chat_completion | non_streaming | test_image_chat_completion_non_streaming | ❌ | +| Text | /chat_completion | non_streaming | test_text_chat_completion_non_streaming | ❌ | +| Text | /chat_completion | tool_calling | test_text_chat_completion_with_tool_calling_and_streaming | ❌ | +| Text | /chat_completion | tool_calling | test_text_chat_completion_with_tool_calling_and_non_streaming | ❌ | +| Text | /completion | streaming | test_text_completion_streaming | ❌ | +| Text | /completion | non_streaming | test_text_completion_non_streaming | ❌ | +| Text | /completion | structured_output | test_text_completion_structured_output | ❌ | + +## Memory: +| API | Capability | Test | Status | +|:-----|:-----|:-----|:-----| +| /insert, /query | inline | test_memory_bank_insert_inline_and_query | ❌ | +| /insert, /query | url | test_memory_bank_insert_from_url_and_query | ❌ | + +## Agents +| API | Capability | Test | Status | +|:-----|:-----|:-----|:-----| +| create_agent_turn | rag | test_rag_agent | ❌ | +| create_agent_turn | custom_tool | test_custom_tool | ❌ | +| create_agent_turn | code_execution | test_code_execution | ❌ | diff --git a/llama_stack/templates/fireworks/report.md b/llama_stack/templates/fireworks/report.md new file mode 100644 index 000000000..00e8f6a55 --- /dev/null +++ b/llama_stack/templates/fireworks/report.md @@ -0,0 +1,44 @@ +# Report for fireworks distribution + +## Supported Models +| Model Descriptor | fireworks | +|:---|:---| +| Llama-3-8B-Instruct | ❌ | +| Llama-3-70B-Instruct | ❌ | +| Llama3.1-8B-Instruct | ✅ | +| Llama3.1-70B-Instruct | ✅ | +| Llama3.1-405B-Instruct | ✅ | +| Llama3.2-1B-Instruct | ✅ | +| Llama3.2-3B-Instruct | ✅ | +| Llama3.2-11B-Vision-Instruct | ✅ | +| Llama3.2-90B-Vision-Instruct | ✅ | +| Llama3.3-70B-Instruct | ✅ | +| Llama-Guard-3-11B-Vision | ✅ | +| Llama-Guard-3-1B | ❌ | +| Llama-Guard-3-8B | ✅ | +| Llama-Guard-2-8B | ❌ | + +## Inference +| Model | API | Capability | Test | Status | +|:----- |:-----|:-----|:-----|:-----| +| Llama-3.1-8B-Instruct | /chat_completion | streaming | test_text_chat_completion_streaming | ✅ | +| Llama-3.2-11B-Vision-Instruct | /chat_completion | streaming | test_image_chat_completion_streaming | ✅ | +| Llama-3.2-11B-Vision-Instruct | /chat_completion | non_streaming | test_image_chat_completion_non_streaming | ✅ | +| Llama-3.1-8B-Instruct | /chat_completion | non_streaming | test_text_chat_completion_non_streaming | ✅ | +| Llama-3.1-8B-Instruct | /chat_completion | tool_calling | test_text_chat_completion_with_tool_calling_and_streaming | ✅ | +| Llama-3.1-8B-Instruct | /chat_completion | tool_calling | test_text_chat_completion_with_tool_calling_and_non_streaming | ✅ | +| Llama-3.1-8B-Instruct | /completion | streaming | test_text_completion_streaming | ✅ | +| Llama-3.1-8B-Instruct | /completion | non_streaming | test_text_completion_non_streaming | ✅ | +| Llama-3.1-8B-Instruct | /completion | structured_output | test_text_completion_structured_output | ✅ | + +## Vector IO +| API | Capability | Test | Status | +|:-----|:-----|:-----|:-----| +| /retrieve | | test_vector_db_retrieve | ✅ | + +## Agents +| API | Capability | Test | Status | +|:-----|:-----|:-----|:-----| +| /create_agent_turn | rag | test_rag_agent | ✅ | +| /create_agent_turn | custom_tool | test_custom_tool | ✅ | +| /create_agent_turn | code_execution | test_code_interpreter_for_attachments | ✅ | diff --git a/llama_stack/templates/fireworks/run-with-safety.yaml b/llama_stack/templates/fireworks/run-with-safety.yaml index 58cdce85d..a4b425436 100644 --- a/llama_stack/templates/fireworks/run-with-safety.yaml +++ b/llama_stack/templates/fireworks/run-with-safety.yaml @@ -1,16 +1,15 @@ version: '2' image_name: fireworks -conda_env: fireworks apis: - agents - datasetio - eval - inference -- memory - safety - scoring - telemetry - tool_runtime +- vector_io providers: inference: - provider_id: fireworks @@ -21,7 +20,7 @@ providers: - provider_id: sentence-transformers provider_type: inline::sentence-transformers config: {} - memory: + vector_io: - provider_id: faiss provider_type: inline::faiss config: @@ -33,6 +32,9 @@ providers: - provider_id: llama-guard provider_type: inline::llama-guard config: {} + - provider_id: llama-guard-vision + provider_type: inline::llama-guard + config: {} - provider_id: code-scanner provider_type: inline::code-scanner config: {} @@ -87,8 +89,11 @@ providers: - provider_id: code-interpreter provider_type: inline::code-interpreter config: {} - - provider_id: memory-runtime - provider_type: inline::memory-runtime + - provider_id: rag-runtime + provider_type: inline::rag-runtime + config: {} + - provider_id: model-context-protocol + provider_type: remote::model-context-protocol config: {} metadata_store: type: sqlite @@ -97,52 +102,52 @@ models: - metadata: {} model_id: meta-llama/Llama-3.1-8B-Instruct provider_id: fireworks - provider_model_id: fireworks/llama-v3p1-8b-instruct + provider_model_id: accounts/fireworks/models/llama-v3p1-8b-instruct model_type: llm - metadata: {} model_id: meta-llama/Llama-3.1-70B-Instruct provider_id: fireworks - provider_model_id: fireworks/llama-v3p1-70b-instruct + provider_model_id: accounts/fireworks/models/llama-v3p1-70b-instruct model_type: llm - metadata: {} model_id: meta-llama/Llama-3.1-405B-Instruct-FP8 provider_id: fireworks - provider_model_id: fireworks/llama-v3p1-405b-instruct + provider_model_id: accounts/fireworks/models/llama-v3p1-405b-instruct model_type: llm - metadata: {} model_id: meta-llama/Llama-3.2-1B-Instruct provider_id: fireworks - provider_model_id: fireworks/llama-v3p2-1b-instruct + provider_model_id: accounts/fireworks/models/llama-v3p2-1b-instruct model_type: llm - metadata: {} model_id: meta-llama/Llama-3.2-3B-Instruct provider_id: fireworks - provider_model_id: fireworks/llama-v3p2-3b-instruct + provider_model_id: accounts/fireworks/models/llama-v3p2-3b-instruct model_type: llm - metadata: {} model_id: meta-llama/Llama-3.2-11B-Vision-Instruct provider_id: fireworks - provider_model_id: fireworks/llama-v3p2-11b-vision-instruct + provider_model_id: accounts/fireworks/models/llama-v3p2-11b-vision-instruct model_type: llm - metadata: {} model_id: meta-llama/Llama-3.2-90B-Vision-Instruct provider_id: fireworks - provider_model_id: fireworks/llama-v3p2-90b-vision-instruct + provider_model_id: accounts/fireworks/models/llama-v3p2-90b-vision-instruct model_type: llm - metadata: {} model_id: meta-llama/Llama-3.3-70B-Instruct provider_id: fireworks - provider_model_id: fireworks/llama-v3p3-70b-instruct + provider_model_id: accounts/fireworks/models/llama-v3p3-70b-instruct model_type: llm - metadata: {} model_id: meta-llama/Llama-Guard-3-8B provider_id: fireworks - provider_model_id: fireworks/llama-guard-3-8b + provider_model_id: accounts/fireworks/models/llama-guard-3-8b model_type: llm - metadata: {} model_id: meta-llama/Llama-Guard-3-11B-Vision provider_id: fireworks - provider_model_id: fireworks/llama-guard-3-11b-vision + provider_model_id: accounts/fireworks/models/llama-guard-3-11b-vision model_type: llm - metadata: embedding_dimension: 384 @@ -152,16 +157,18 @@ models: shields: - shield_id: meta-llama/Llama-Guard-3-8B provider_id: llama-guard +- shield_id: meta-llama/Llama-Guard-3-11B-Vision + provider_id: llama-guard-vision - shield_id: CodeScanner provider_id: code-scanner -memory_banks: [] +vector_dbs: [] datasets: [] scoring_fns: [] eval_tasks: [] tool_groups: - toolgroup_id: builtin::websearch provider_id: tavily-search -- toolgroup_id: builtin::memory - provider_id: memory-runtime +- toolgroup_id: builtin::rag + provider_id: rag-runtime - toolgroup_id: builtin::code_interpreter provider_id: code-interpreter diff --git a/llama_stack/templates/fireworks/run.yaml b/llama_stack/templates/fireworks/run.yaml index 6c41b3ed7..a497317bd 100644 --- a/llama_stack/templates/fireworks/run.yaml +++ b/llama_stack/templates/fireworks/run.yaml @@ -1,16 +1,15 @@ version: '2' image_name: fireworks -conda_env: fireworks apis: - agents - datasetio - eval - inference -- memory - safety - scoring - telemetry - tool_runtime +- vector_io providers: inference: - provider_id: fireworks @@ -21,7 +20,7 @@ providers: - provider_id: sentence-transformers provider_type: inline::sentence-transformers config: {} - memory: + vector_io: - provider_id: faiss provider_type: inline::faiss config: @@ -47,7 +46,7 @@ providers: config: service_name: ${env.OTEL_SERVICE_NAME:llama-stack} sinks: ${env.TELEMETRY_SINKS:console,sqlite} - sqlite_db_path: ${env.SQLITE_DB_PATH:~/.llama/distributions/accounts/fireworks/models/trace_store.db} + sqlite_db_path: ${env.SQLITE_DB_PATH:~/.llama/distributions/fireworks/trace_store.db} eval: - provider_id: meta-reference provider_type: inline::meta-reference @@ -84,8 +83,11 @@ providers: - provider_id: code-interpreter provider_type: inline::code-interpreter config: {} - - provider_id: memory-runtime - provider_type: inline::memory-runtime + - provider_id: rag-runtime + provider_type: inline::rag-runtime + config: {} + - provider_id: model-context-protocol + provider_type: remote::model-context-protocol config: {} metadata_store: type: sqlite @@ -148,14 +150,14 @@ models: model_type: embedding shields: - shield_id: meta-llama/Llama-Guard-3-8B -memory_banks: [] +vector_dbs: [] datasets: [] scoring_fns: [] eval_tasks: [] tool_groups: - toolgroup_id: builtin::websearch provider_id: tavily-search -- toolgroup_id: builtin::memory - provider_id: memory-runtime +- toolgroup_id: builtin::rag + provider_id: rag-runtime - toolgroup_id: builtin::code_interpreter provider_id: code-interpreter diff --git a/llama_stack/templates/hf-endpoint/build.yaml b/llama_stack/templates/hf-endpoint/build.yaml index c18689855..c2eaaa05b 100644 --- a/llama_stack/templates/hf-endpoint/build.yaml +++ b/llama_stack/templates/hf-endpoint/build.yaml @@ -1,11 +1,10 @@ version: '2' -name: hf-endpoint distribution_spec: description: Use (an external) Hugging Face Inference Endpoint for running LLM inference providers: inference: - remote::hf::endpoint - memory: + vector_io: - inline::faiss - remote::chromadb - remote::pgvector @@ -28,5 +27,6 @@ distribution_spec: - remote::brave-search - remote::tavily-search - inline::code-interpreter - - inline::memory-runtime + - inline::rag-runtime + - remote::model-context-protocol image_type: conda diff --git a/llama_stack/templates/hf-endpoint/hf_endpoint.py b/llama_stack/templates/hf-endpoint/hf_endpoint.py index 54aaa56ac..4533fd95b 100644 --- a/llama_stack/templates/hf-endpoint/hf_endpoint.py +++ b/llama_stack/templates/hf-endpoint/hf_endpoint.py @@ -14,7 +14,7 @@ from llama_stack.distribution.datatypes import ( from llama_stack.providers.inline.inference.sentence_transformers import ( SentenceTransformersInferenceConfig, ) -from llama_stack.providers.inline.memory.faiss.config import FaissImplConfig +from llama_stack.providers.inline.vector_io.faiss.config import FaissImplConfig from llama_stack.providers.remote.inference.tgi import InferenceEndpointImplConfig from llama_stack.templates.template import DistributionTemplate, RunConfigSettings @@ -22,7 +22,7 @@ from llama_stack.templates.template import DistributionTemplate, RunConfigSettin def get_distribution_template() -> DistributionTemplate: providers = { "inference": ["remote::hf::endpoint"], - "memory": ["inline::faiss", "remote::chromadb", "remote::pgvector"], + "vector_io": ["inline::faiss", "remote::chromadb", "remote::pgvector"], "safety": ["inline::llama-guard"], "agents": ["inline::meta-reference"], "telemetry": ["inline::meta-reference"], @@ -33,7 +33,8 @@ def get_distribution_template() -> DistributionTemplate: "remote::brave-search", "remote::tavily-search", "inline::code-interpreter", - "inline::memory-runtime", + "inline::rag-runtime", + "remote::model-context-protocol", ], } name = "hf-endpoint" @@ -47,7 +48,7 @@ def get_distribution_template() -> DistributionTemplate: provider_type="inline::sentence-transformers", config=SentenceTransformersInferenceConfig.sample_run_config(), ) - memory_provider = Provider( + vector_io_provider = Provider( provider_id="faiss", provider_type="inline::faiss", config=FaissImplConfig.sample_run_config(f"distributions/{name}"), @@ -75,8 +76,8 @@ def get_distribution_template() -> DistributionTemplate: provider_id="tavily-search", ), ToolGroupInput( - toolgroup_id="builtin::memory", - provider_id="memory-runtime", + toolgroup_id="builtin::rag", + provider_id="rag-runtime", ), ToolGroupInput( toolgroup_id="builtin::code_interpreter", @@ -88,7 +89,7 @@ def get_distribution_template() -> DistributionTemplate: name=name, distro_type="self_hosted", description="Use (an external) Hugging Face Inference Endpoint for running LLM inference", - docker_image=None, + container_image=None, template_path=None, providers=providers, default_models=[inference_model, safety_model], @@ -96,7 +97,7 @@ def get_distribution_template() -> DistributionTemplate: "run.yaml": RunConfigSettings( provider_overrides={ "inference": [inference_provider, embedding_provider], - "memory": [memory_provider], + "vector_io": [vector_io_provider], }, default_models=[inference_model, embedding_model], default_tool_groups=default_tool_groups, @@ -114,7 +115,7 @@ def get_distribution_template() -> DistributionTemplate: ), ), ], - "memory": [memory_provider], + "vector_io": [vector_io_provider], }, default_models=[ inference_model, diff --git a/llama_stack/templates/hf-endpoint/run-with-safety.yaml b/llama_stack/templates/hf-endpoint/run-with-safety.yaml index a9d895d23..0329f580b 100644 --- a/llama_stack/templates/hf-endpoint/run-with-safety.yaml +++ b/llama_stack/templates/hf-endpoint/run-with-safety.yaml @@ -1,16 +1,15 @@ version: '2' image_name: hf-endpoint -conda_env: hf-endpoint apis: - agents - datasetio - eval - inference -- memory - safety - scoring - telemetry - tool_runtime +- vector_io providers: inference: - provider_id: hf-endpoint @@ -26,7 +25,7 @@ providers: config: endpoint_name: ${env.SAFETY_INFERENCE_ENDPOINT_NAME} api_token: ${env.HF_API_TOKEN} - memory: + vector_io: - provider_id: faiss provider_type: inline::faiss config: @@ -89,8 +88,11 @@ providers: - provider_id: code-interpreter provider_type: inline::code-interpreter config: {} - - provider_id: memory-runtime - provider_type: inline::memory-runtime + - provider_id: rag-runtime + provider_type: inline::rag-runtime + config: {} + - provider_id: model-context-protocol + provider_type: remote::model-context-protocol config: {} metadata_store: type: sqlite @@ -111,14 +113,14 @@ models: model_type: embedding shields: - shield_id: ${env.SAFETY_MODEL} -memory_banks: [] +vector_dbs: [] datasets: [] scoring_fns: [] eval_tasks: [] tool_groups: - toolgroup_id: builtin::websearch provider_id: tavily-search -- toolgroup_id: builtin::memory - provider_id: memory-runtime +- toolgroup_id: builtin::rag + provider_id: rag-runtime - toolgroup_id: builtin::code_interpreter provider_id: code-interpreter diff --git a/llama_stack/templates/hf-endpoint/run.yaml b/llama_stack/templates/hf-endpoint/run.yaml index e9b58c962..8163fe28e 100644 --- a/llama_stack/templates/hf-endpoint/run.yaml +++ b/llama_stack/templates/hf-endpoint/run.yaml @@ -1,16 +1,15 @@ version: '2' image_name: hf-endpoint -conda_env: hf-endpoint apis: - agents - datasetio - eval - inference -- memory - safety - scoring - telemetry - tool_runtime +- vector_io providers: inference: - provider_id: hf-endpoint @@ -21,7 +20,7 @@ providers: - provider_id: sentence-transformers provider_type: inline::sentence-transformers config: {} - memory: + vector_io: - provider_id: faiss provider_type: inline::faiss config: @@ -84,8 +83,11 @@ providers: - provider_id: code-interpreter provider_type: inline::code-interpreter config: {} - - provider_id: memory-runtime - provider_type: inline::memory-runtime + - provider_id: rag-runtime + provider_type: inline::rag-runtime + config: {} + - provider_id: model-context-protocol + provider_type: remote::model-context-protocol config: {} metadata_store: type: sqlite @@ -101,14 +103,14 @@ models: provider_id: sentence-transformers model_type: embedding shields: [] -memory_banks: [] +vector_dbs: [] datasets: [] scoring_fns: [] eval_tasks: [] tool_groups: - toolgroup_id: builtin::websearch provider_id: tavily-search -- toolgroup_id: builtin::memory - provider_id: memory-runtime +- toolgroup_id: builtin::rag + provider_id: rag-runtime - toolgroup_id: builtin::code_interpreter provider_id: code-interpreter diff --git a/llama_stack/templates/hf-serverless/build.yaml b/llama_stack/templates/hf-serverless/build.yaml index a6b551e4a..f9303cfab 100644 --- a/llama_stack/templates/hf-serverless/build.yaml +++ b/llama_stack/templates/hf-serverless/build.yaml @@ -1,11 +1,10 @@ version: '2' -name: hf-serverless distribution_spec: description: Use (an external) Hugging Face Inference Endpoint for running LLM inference providers: inference: - remote::hf::serverless - memory: + vector_io: - inline::faiss - remote::chromadb - remote::pgvector @@ -28,5 +27,6 @@ distribution_spec: - remote::brave-search - remote::tavily-search - inline::code-interpreter - - inline::memory-runtime + - inline::rag-runtime + - remote::model-context-protocol image_type: conda diff --git a/llama_stack/templates/hf-serverless/hf_serverless.py b/llama_stack/templates/hf-serverless/hf_serverless.py index 51e16c3db..8438de7a5 100644 --- a/llama_stack/templates/hf-serverless/hf_serverless.py +++ b/llama_stack/templates/hf-serverless/hf_serverless.py @@ -14,7 +14,7 @@ from llama_stack.distribution.datatypes import ( from llama_stack.providers.inline.inference.sentence_transformers import ( SentenceTransformersInferenceConfig, ) -from llama_stack.providers.inline.memory.faiss.config import FaissImplConfig +from llama_stack.providers.inline.vector_io.faiss.config import FaissImplConfig from llama_stack.providers.remote.inference.tgi import InferenceAPIImplConfig from llama_stack.templates.template import DistributionTemplate, RunConfigSettings @@ -22,7 +22,7 @@ from llama_stack.templates.template import DistributionTemplate, RunConfigSettin def get_distribution_template() -> DistributionTemplate: providers = { "inference": ["remote::hf::serverless"], - "memory": ["inline::faiss", "remote::chromadb", "remote::pgvector"], + "vector_io": ["inline::faiss", "remote::chromadb", "remote::pgvector"], "safety": ["inline::llama-guard"], "agents": ["inline::meta-reference"], "telemetry": ["inline::meta-reference"], @@ -33,7 +33,8 @@ def get_distribution_template() -> DistributionTemplate: "remote::brave-search", "remote::tavily-search", "inline::code-interpreter", - "inline::memory-runtime", + "inline::rag-runtime", + "remote::model-context-protocol", ], } @@ -48,7 +49,7 @@ def get_distribution_template() -> DistributionTemplate: provider_type="inline::sentence-transformers", config=SentenceTransformersInferenceConfig.sample_run_config(), ) - memory_provider = Provider( + vector_io_provider = Provider( provider_id="faiss", provider_type="inline::faiss", config=FaissImplConfig.sample_run_config(f"distributions/{name}"), @@ -76,8 +77,8 @@ def get_distribution_template() -> DistributionTemplate: provider_id="tavily-search", ), ToolGroupInput( - toolgroup_id="builtin::memory", - provider_id="memory-runtime", + toolgroup_id="builtin::rag", + provider_id="rag-runtime", ), ToolGroupInput( toolgroup_id="builtin::code_interpreter", @@ -89,7 +90,7 @@ def get_distribution_template() -> DistributionTemplate: name=name, distro_type="self_hosted", description="Use (an external) Hugging Face Inference Endpoint for running LLM inference", - docker_image=None, + container_image=None, template_path=None, providers=providers, default_models=[inference_model, safety_model], @@ -97,9 +98,10 @@ def get_distribution_template() -> DistributionTemplate: "run.yaml": RunConfigSettings( provider_overrides={ "inference": [inference_provider, embedding_provider], - "memory": [memory_provider], + "vector_io": [vector_io_provider], }, default_models=[inference_model, embedding_model], + default_tool_groups=default_tool_groups, ), "run-with-safety.yaml": RunConfigSettings( provider_overrides={ @@ -114,7 +116,7 @@ def get_distribution_template() -> DistributionTemplate: ), ), ], - "memory": [memory_provider], + "vector_io": [vector_io_provider], }, default_models=[ inference_model, diff --git a/llama_stack/templates/hf-serverless/run-with-safety.yaml b/llama_stack/templates/hf-serverless/run-with-safety.yaml index 415cec648..9cee920a5 100644 --- a/llama_stack/templates/hf-serverless/run-with-safety.yaml +++ b/llama_stack/templates/hf-serverless/run-with-safety.yaml @@ -1,16 +1,15 @@ version: '2' image_name: hf-serverless -conda_env: hf-serverless apis: - agents - datasetio - eval - inference -- memory - safety - scoring - telemetry - tool_runtime +- vector_io providers: inference: - provider_id: hf-serverless @@ -26,7 +25,7 @@ providers: config: huggingface_repo: ${env.SAFETY_MODEL} api_token: ${env.HF_API_TOKEN} - memory: + vector_io: - provider_id: faiss provider_type: inline::faiss config: @@ -89,8 +88,11 @@ providers: - provider_id: code-interpreter provider_type: inline::code-interpreter config: {} - - provider_id: memory-runtime - provider_type: inline::memory-runtime + - provider_id: rag-runtime + provider_type: inline::rag-runtime + config: {} + - provider_id: model-context-protocol + provider_type: remote::model-context-protocol config: {} metadata_store: type: sqlite @@ -111,14 +113,14 @@ models: model_type: embedding shields: - shield_id: ${env.SAFETY_MODEL} -memory_banks: [] +vector_dbs: [] datasets: [] scoring_fns: [] eval_tasks: [] tool_groups: - toolgroup_id: builtin::websearch provider_id: tavily-search -- toolgroup_id: builtin::memory - provider_id: memory-runtime +- toolgroup_id: builtin::rag + provider_id: rag-runtime - toolgroup_id: builtin::code_interpreter provider_id: code-interpreter diff --git a/llama_stack/templates/hf-serverless/run.yaml b/llama_stack/templates/hf-serverless/run.yaml index ef9dedeed..c8ad0d38d 100644 --- a/llama_stack/templates/hf-serverless/run.yaml +++ b/llama_stack/templates/hf-serverless/run.yaml @@ -1,16 +1,15 @@ version: '2' image_name: hf-serverless -conda_env: hf-serverless apis: - agents - datasetio - eval - inference -- memory - safety - scoring - telemetry - tool_runtime +- vector_io providers: inference: - provider_id: hf-serverless @@ -21,7 +20,7 @@ providers: - provider_id: sentence-transformers provider_type: inline::sentence-transformers config: {} - memory: + vector_io: - provider_id: faiss provider_type: inline::faiss config: @@ -84,8 +83,11 @@ providers: - provider_id: code-interpreter provider_type: inline::code-interpreter config: {} - - provider_id: memory-runtime - provider_type: inline::memory-runtime + - provider_id: rag-runtime + provider_type: inline::rag-runtime + config: {} + - provider_id: model-context-protocol + provider_type: remote::model-context-protocol config: {} metadata_store: type: sqlite @@ -101,8 +103,14 @@ models: provider_id: sentence-transformers model_type: embedding shields: [] -memory_banks: [] +vector_dbs: [] datasets: [] scoring_fns: [] eval_tasks: [] -tool_groups: [] +tool_groups: +- toolgroup_id: builtin::websearch + provider_id: tavily-search +- toolgroup_id: builtin::rag + provider_id: rag-runtime +- toolgroup_id: builtin::code_interpreter + provider_id: code-interpreter diff --git a/llama_stack/templates/meta-reference-gpu/build.yaml b/llama_stack/templates/meta-reference-gpu/build.yaml index ba8413fa6..b9130fc7d 100644 --- a/llama_stack/templates/meta-reference-gpu/build.yaml +++ b/llama_stack/templates/meta-reference-gpu/build.yaml @@ -1,11 +1,10 @@ version: '2' -name: meta-reference-gpu distribution_spec: description: Use Meta Reference for running LLM inference providers: inference: - inline::meta-reference - memory: + vector_io: - inline::faiss - remote::chromadb - remote::pgvector @@ -28,5 +27,6 @@ distribution_spec: - remote::brave-search - remote::tavily-search - inline::code-interpreter - - inline::memory-runtime + - inline::rag-runtime + - remote::model-context-protocol image_type: conda diff --git a/llama_stack/templates/meta-reference-gpu/meta_reference.py b/llama_stack/templates/meta-reference-gpu/meta_reference.py index 1477b31ff..a3f82b0c8 100644 --- a/llama_stack/templates/meta-reference-gpu/meta_reference.py +++ b/llama_stack/templates/meta-reference-gpu/meta_reference.py @@ -19,14 +19,14 @@ from llama_stack.providers.inline.inference.meta_reference import ( from llama_stack.providers.inline.inference.sentence_transformers import ( SentenceTransformersInferenceConfig, ) -from llama_stack.providers.inline.memory.faiss.config import FaissImplConfig +from llama_stack.providers.inline.vector_io.faiss.config import FaissImplConfig from llama_stack.templates.template import DistributionTemplate, RunConfigSettings def get_distribution_template() -> DistributionTemplate: providers = { "inference": ["inline::meta-reference"], - "memory": ["inline::faiss", "remote::chromadb", "remote::pgvector"], + "vector_io": ["inline::faiss", "remote::chromadb", "remote::pgvector"], "safety": ["inline::llama-guard"], "agents": ["inline::meta-reference"], "telemetry": ["inline::meta-reference"], @@ -37,7 +37,8 @@ def get_distribution_template() -> DistributionTemplate: "remote::brave-search", "remote::tavily-search", "inline::code-interpreter", - "inline::memory-runtime", + "inline::rag-runtime", + "remote::model-context-protocol", ], } name = "meta-reference-gpu" @@ -54,7 +55,7 @@ def get_distribution_template() -> DistributionTemplate: provider_type="inline::sentence-transformers", config=SentenceTransformersInferenceConfig.sample_run_config(), ) - memory_provider = Provider( + vector_io_provider = Provider( provider_id="faiss", provider_type="inline::faiss", config=FaissImplConfig.sample_run_config(f"distributions/{name}"), @@ -82,8 +83,8 @@ def get_distribution_template() -> DistributionTemplate: provider_id="tavily-search", ), ToolGroupInput( - toolgroup_id="builtin::memory", - provider_id="memory-runtime", + toolgroup_id="builtin::rag", + provider_id="rag-runtime", ), ToolGroupInput( toolgroup_id="builtin::code_interpreter", @@ -102,9 +103,10 @@ def get_distribution_template() -> DistributionTemplate: "run.yaml": RunConfigSettings( provider_overrides={ "inference": [inference_provider, embedding_provider], - "memory": [memory_provider], + "vector_io": [vector_io_provider], }, default_models=[inference_model, embedding_model], + default_tool_groups=default_tool_groups, ), "run-with-safety.yaml": RunConfigSettings( provider_overrides={ @@ -120,7 +122,7 @@ def get_distribution_template() -> DistributionTemplate: ), ), ], - "memory": [memory_provider], + "vector_io": [vector_io_provider], }, default_models=[ inference_model, diff --git a/llama_stack/templates/meta-reference-gpu/run-with-safety.yaml b/llama_stack/templates/meta-reference-gpu/run-with-safety.yaml index 4946fdab7..0faaabb15 100644 --- a/llama_stack/templates/meta-reference-gpu/run-with-safety.yaml +++ b/llama_stack/templates/meta-reference-gpu/run-with-safety.yaml @@ -1,16 +1,15 @@ version: '2' image_name: meta-reference-gpu -conda_env: meta-reference-gpu apis: - agents - datasetio - eval - inference -- memory - safety - scoring - telemetry - tool_runtime +- vector_io providers: inference: - provider_id: meta-reference-inference @@ -28,7 +27,7 @@ providers: model: ${env.SAFETY_MODEL} max_seq_len: 4096 checkpoint_dir: ${env.SAFETY_CHECKPOINT_DIR:null} - memory: + vector_io: - provider_id: faiss provider_type: inline::faiss config: @@ -91,8 +90,11 @@ providers: - provider_id: code-interpreter provider_type: inline::code-interpreter config: {} - - provider_id: memory-runtime - provider_type: inline::memory-runtime + - provider_id: rag-runtime + provider_type: inline::rag-runtime + config: {} + - provider_id: model-context-protocol + provider_type: remote::model-context-protocol config: {} metadata_store: type: sqlite @@ -113,14 +115,14 @@ models: model_type: embedding shields: - shield_id: ${env.SAFETY_MODEL} -memory_banks: [] +vector_dbs: [] datasets: [] scoring_fns: [] eval_tasks: [] tool_groups: - toolgroup_id: builtin::websearch provider_id: tavily-search -- toolgroup_id: builtin::memory - provider_id: memory-runtime +- toolgroup_id: builtin::rag + provider_id: rag-runtime - toolgroup_id: builtin::code_interpreter provider_id: code-interpreter diff --git a/llama_stack/templates/meta-reference-gpu/run.yaml b/llama_stack/templates/meta-reference-gpu/run.yaml index 52345f3c1..6ffe1fa36 100644 --- a/llama_stack/templates/meta-reference-gpu/run.yaml +++ b/llama_stack/templates/meta-reference-gpu/run.yaml @@ -1,16 +1,15 @@ version: '2' image_name: meta-reference-gpu -conda_env: meta-reference-gpu apis: - agents - datasetio - eval - inference -- memory - safety - scoring - telemetry - tool_runtime +- vector_io providers: inference: - provider_id: meta-reference-inference @@ -22,7 +21,7 @@ providers: - provider_id: sentence-transformers provider_type: inline::sentence-transformers config: {} - memory: + vector_io: - provider_id: faiss provider_type: inline::faiss config: @@ -85,8 +84,11 @@ providers: - provider_id: code-interpreter provider_type: inline::code-interpreter config: {} - - provider_id: memory-runtime - provider_type: inline::memory-runtime + - provider_id: rag-runtime + provider_type: inline::rag-runtime + config: {} + - provider_id: model-context-protocol + provider_type: remote::model-context-protocol config: {} metadata_store: type: sqlite @@ -102,8 +104,14 @@ models: provider_id: sentence-transformers model_type: embedding shields: [] -memory_banks: [] +vector_dbs: [] datasets: [] scoring_fns: [] eval_tasks: [] -tool_groups: [] +tool_groups: +- toolgroup_id: builtin::websearch + provider_id: tavily-search +- toolgroup_id: builtin::rag + provider_id: rag-runtime +- toolgroup_id: builtin::code_interpreter + provider_id: code-interpreter diff --git a/llama_stack/templates/meta-reference-quantized-gpu/build.yaml b/llama_stack/templates/meta-reference-quantized-gpu/build.yaml index 41ab44e38..7bbcfe5f2 100644 --- a/llama_stack/templates/meta-reference-quantized-gpu/build.yaml +++ b/llama_stack/templates/meta-reference-quantized-gpu/build.yaml @@ -1,11 +1,10 @@ version: '2' -name: meta-reference-quantized-gpu distribution_spec: description: Use Meta Reference with fp8, int4 quantization for running LLM inference providers: inference: - inline::meta-reference-quantized - memory: + vector_io: - inline::faiss - remote::chromadb - remote::pgvector @@ -28,5 +27,6 @@ distribution_spec: - remote::brave-search - remote::tavily-search - inline::code-interpreter - - inline::memory-runtime + - inline::rag-runtime + - remote::model-context-protocol image_type: conda diff --git a/llama_stack/templates/meta-reference-quantized-gpu/meta_reference.py b/llama_stack/templates/meta-reference-quantized-gpu/meta_reference.py index 5c40134af..8c2a6ec9f 100644 --- a/llama_stack/templates/meta-reference-quantized-gpu/meta_reference.py +++ b/llama_stack/templates/meta-reference-quantized-gpu/meta_reference.py @@ -14,14 +14,14 @@ from llama_stack.providers.inline.inference.meta_reference import ( from llama_stack.providers.inline.inference.sentence_transformers import ( SentenceTransformersInferenceConfig, ) -from llama_stack.providers.inline.memory.faiss.config import FaissImplConfig +from llama_stack.providers.inline.vector_io.faiss.config import FaissImplConfig from llama_stack.templates.template import DistributionTemplate, RunConfigSettings def get_distribution_template() -> DistributionTemplate: providers = { "inference": ["inline::meta-reference-quantized"], - "memory": ["inline::faiss", "remote::chromadb", "remote::pgvector"], + "vector_io": ["inline::faiss", "remote::chromadb", "remote::pgvector"], "safety": ["inline::llama-guard"], "agents": ["inline::meta-reference"], "telemetry": ["inline::meta-reference"], @@ -32,7 +32,8 @@ def get_distribution_template() -> DistributionTemplate: "remote::brave-search", "remote::tavily-search", "inline::code-interpreter", - "inline::memory-runtime", + "inline::rag-runtime", + "remote::model-context-protocol", ], } default_tool_groups = [ @@ -41,8 +42,8 @@ def get_distribution_template() -> DistributionTemplate: provider_id="tavily-search", ), ToolGroupInput( - toolgroup_id="builtin::memory", - provider_id="memory-runtime", + toolgroup_id="builtin::rag", + provider_id="rag-runtime", ), ToolGroupInput( toolgroup_id="builtin::code_interpreter", @@ -63,7 +64,7 @@ def get_distribution_template() -> DistributionTemplate: provider_type="inline::sentence-transformers", config=SentenceTransformersInferenceConfig.sample_run_config(), ) - memory_provider = Provider( + vector_io_provider = Provider( provider_id="faiss", provider_type="inline::faiss", config=FaissImplConfig.sample_run_config(f"distributions/{name}"), @@ -92,7 +93,7 @@ def get_distribution_template() -> DistributionTemplate: "run.yaml": RunConfigSettings( provider_overrides={ "inference": [inference_provider, embedding_provider], - "memory": [memory_provider], + "vector_io": [vector_io_provider], }, default_models=[inference_model, embedding_model], default_tool_groups=default_tool_groups, diff --git a/llama_stack/templates/meta-reference-quantized-gpu/run.yaml b/llama_stack/templates/meta-reference-quantized-gpu/run.yaml index 02a5bacaa..5ff87a901 100644 --- a/llama_stack/templates/meta-reference-quantized-gpu/run.yaml +++ b/llama_stack/templates/meta-reference-quantized-gpu/run.yaml @@ -1,16 +1,15 @@ version: '2' image_name: meta-reference-quantized-gpu -conda_env: meta-reference-quantized-gpu apis: - agents - datasetio - eval - inference -- memory - safety - scoring - telemetry - tool_runtime +- vector_io providers: inference: - provider_id: meta-reference-inference @@ -24,7 +23,7 @@ providers: - provider_id: sentence-transformers provider_type: inline::sentence-transformers config: {} - memory: + vector_io: - provider_id: faiss provider_type: inline::faiss config: @@ -87,8 +86,11 @@ providers: - provider_id: code-interpreter provider_type: inline::code-interpreter config: {} - - provider_id: memory-runtime - provider_type: inline::memory-runtime + - provider_id: rag-runtime + provider_type: inline::rag-runtime + config: {} + - provider_id: model-context-protocol + provider_type: remote::model-context-protocol config: {} metadata_store: type: sqlite @@ -104,14 +106,14 @@ models: provider_id: sentence-transformers model_type: embedding shields: [] -memory_banks: [] +vector_dbs: [] datasets: [] scoring_fns: [] eval_tasks: [] tool_groups: - toolgroup_id: builtin::websearch provider_id: tavily-search -- toolgroup_id: builtin::memory - provider_id: memory-runtime +- toolgroup_id: builtin::rag + provider_id: rag-runtime - toolgroup_id: builtin::code_interpreter provider_id: code-interpreter diff --git a/llama_stack/templates/nvidia/__init__.py b/llama_stack/templates/nvidia/__init__.py new file mode 100644 index 000000000..24e2fbd21 --- /dev/null +++ b/llama_stack/templates/nvidia/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +from .nvidia import get_distribution_template # noqa: F401 diff --git a/llama_stack/templates/nvidia/build.yaml b/llama_stack/templates/nvidia/build.yaml new file mode 100644 index 000000000..e9748721a --- /dev/null +++ b/llama_stack/templates/nvidia/build.yaml @@ -0,0 +1,30 @@ +version: '2' +distribution_spec: + description: Use NVIDIA NIM for running LLM inference + providers: + inference: + - remote::nvidia + vector_io: + - inline::faiss + safety: + - inline::llama-guard + agents: + - inline::meta-reference + telemetry: + - inline::meta-reference + eval: + - inline::meta-reference + datasetio: + - remote::huggingface + - inline::localfs + scoring: + - inline::basic + - inline::llm-as-judge + - inline::braintrust + tool_runtime: + - remote::brave-search + - remote::tavily-search + - inline::code-interpreter + - inline::rag-runtime + - remote::model-context-protocol +image_type: conda diff --git a/llama_stack/templates/nvidia/doc_template.md b/llama_stack/templates/nvidia/doc_template.md new file mode 100644 index 000000000..9d9006a27 --- /dev/null +++ b/llama_stack/templates/nvidia/doc_template.md @@ -0,0 +1,61 @@ +# NVIDIA Distribution + +The `llamastack/distribution-{{ name }}` distribution consists of the following provider configurations. + +{{ providers_table }} + +{% if run_config_env_vars %} +### Environment Variables + +The following environment variables can be configured: + +{% for var, (default_value, description) in run_config_env_vars.items() %} +- `{{ var }}`: {{ description }} (default: `{{ default_value }}`) +{% endfor %} +{% endif %} + +{% if default_models %} +### Models + +The following models are available by default: + +{% for model in default_models %} +- `{{ model.model_id }} ({{ model.provider_model_id }})` +{% endfor %} +{% endif %} + + +### Prerequisite: API Keys + +Make sure you have access to a NVIDIA API Key. You can get one by visiting [https://build.nvidia.com/](https://build.nvidia.com/). + + +## Running Llama Stack with NVIDIA + +You can do this via Conda (build code) or Docker which has a pre-built image. + +### Via Docker + +This method allows you to get started quickly without having to build the distribution code. + +```bash +LLAMA_STACK_PORT=5001 +docker run \ + -it \ + -p $LLAMA_STACK_PORT:$LLAMA_STACK_PORT \ + -v ./run.yaml:/root/my-run.yaml \ + llamastack/distribution-{{ name }} \ + --yaml-config /root/my-run.yaml \ + --port $LLAMA_STACK_PORT \ + --env NVIDIA_API_KEY=$NVIDIA_API_KEY +``` + +### Via Conda + +```bash +llama stack build --template nvidia --image-type conda +llama stack run ./run.yaml \ + --port 5001 \ + --env NVIDIA_API_KEY=$NVIDIA_API_KEY + --env INFERENCE_MODEL=$INFERENCE_MODEL +``` diff --git a/llama_stack/templates/nvidia/nvidia.py b/llama_stack/templates/nvidia/nvidia.py new file mode 100644 index 000000000..19eb4bd5d --- /dev/null +++ b/llama_stack/templates/nvidia/nvidia.py @@ -0,0 +1,95 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +from pathlib import Path + +from llama_models.sku_list import all_registered_models + +from llama_stack.distribution.datatypes import ModelInput, Provider, ToolGroupInput +from llama_stack.providers.remote.inference.nvidia import NVIDIAConfig +from llama_stack.providers.remote.inference.nvidia.nvidia import _MODEL_ALIASES +from llama_stack.templates.template import DistributionTemplate, RunConfigSettings + + +def get_distribution_template() -> DistributionTemplate: + providers = { + "inference": ["remote::nvidia"], + "vector_io": ["inline::faiss"], + "safety": ["inline::llama-guard"], + "agents": ["inline::meta-reference"], + "telemetry": ["inline::meta-reference"], + "eval": ["inline::meta-reference"], + "datasetio": ["remote::huggingface", "inline::localfs"], + "scoring": ["inline::basic", "inline::llm-as-judge", "inline::braintrust"], + "tool_runtime": [ + "remote::brave-search", + "remote::tavily-search", + "inline::code-interpreter", + "inline::rag-runtime", + "remote::model-context-protocol", + ], + } + + inference_provider = Provider( + provider_id="nvidia", + provider_type="remote::nvidia", + config=NVIDIAConfig.sample_run_config(), + ) + + core_model_to_hf_repo = { + m.descriptor(): m.huggingface_repo for m in all_registered_models() + } + default_models = [ + ModelInput( + model_id=core_model_to_hf_repo[m.llama_model], + provider_model_id=m.provider_model_id, + provider_id="nvidia", + ) + for m in _MODEL_ALIASES + ] + default_tool_groups = [ + ToolGroupInput( + toolgroup_id="builtin::websearch", + provider_id="tavily-search", + ), + ToolGroupInput( + toolgroup_id="builtin::rag", + provider_id="rag-runtime", + ), + ToolGroupInput( + toolgroup_id="builtin::code_interpreter", + provider_id="code-interpreter", + ), + ] + + return DistributionTemplate( + name="nvidia", + distro_type="remote_hosted", + description="Use NVIDIA NIM for running LLM inference", + container_image=None, + template_path=Path(__file__).parent / "doc_template.md", + providers=providers, + default_models=default_models, + run_configs={ + "run.yaml": RunConfigSettings( + provider_overrides={ + "inference": [inference_provider], + }, + default_models=default_models, + default_tool_groups=default_tool_groups, + ), + }, + run_config_env_vars={ + "LLAMASTACK_PORT": ( + "5001", + "Port for the Llama Stack distribution server", + ), + "NVIDIA_API_KEY": ( + "", + "NVIDIA API Key", + ), + }, + ) diff --git a/llama_stack/templates/nvidia/run.yaml b/llama_stack/templates/nvidia/run.yaml new file mode 100644 index 000000000..c57ca2b9a --- /dev/null +++ b/llama_stack/templates/nvidia/run.yaml @@ -0,0 +1,149 @@ +version: '2' +image_name: nvidia +apis: +- agents +- datasetio +- eval +- inference +- safety +- scoring +- telemetry +- tool_runtime +- vector_io +providers: + inference: + - provider_id: nvidia + provider_type: remote::nvidia + config: + url: https://integrate.api.nvidia.com + api_key: ${env.NVIDIA_API_KEY} + vector_io: + - provider_id: faiss + provider_type: inline::faiss + config: + kvstore: + type: sqlite + namespace: null + db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/nvidia}/faiss_store.db + safety: + - provider_id: llama-guard + provider_type: inline::llama-guard + config: {} + agents: + - provider_id: meta-reference + provider_type: inline::meta-reference + config: + persistence_store: + type: sqlite + namespace: null + db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/nvidia}/agents_store.db + telemetry: + - provider_id: meta-reference + provider_type: inline::meta-reference + config: + service_name: ${env.OTEL_SERVICE_NAME:llama-stack} + sinks: ${env.TELEMETRY_SINKS:console,sqlite} + sqlite_db_path: ${env.SQLITE_DB_PATH:~/.llama/distributions/nvidia/trace_store.db} + eval: + - provider_id: meta-reference + provider_type: inline::meta-reference + config: {} + datasetio: + - provider_id: huggingface + provider_type: remote::huggingface + config: {} + - provider_id: localfs + provider_type: inline::localfs + config: {} + scoring: + - provider_id: basic + provider_type: inline::basic + config: {} + - provider_id: llm-as-judge + provider_type: inline::llm-as-judge + config: {} + - provider_id: braintrust + provider_type: inline::braintrust + config: + openai_api_key: ${env.OPENAI_API_KEY:} + tool_runtime: + - provider_id: brave-search + provider_type: remote::brave-search + config: + api_key: ${env.BRAVE_SEARCH_API_KEY:} + max_results: 3 + - provider_id: tavily-search + provider_type: remote::tavily-search + config: + api_key: ${env.TAVILY_SEARCH_API_KEY:} + max_results: 3 + - provider_id: code-interpreter + provider_type: inline::code-interpreter + config: {} + - provider_id: rag-runtime + provider_type: inline::rag-runtime + config: {} + - provider_id: model-context-protocol + provider_type: remote::model-context-protocol + config: {} +metadata_store: + type: sqlite + db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/nvidia}/registry.db +models: +- metadata: {} + model_id: meta-llama/Llama-3-8B-Instruct + provider_id: nvidia + provider_model_id: meta/llama3-8b-instruct + model_type: llm +- metadata: {} + model_id: meta-llama/Llama-3-70B-Instruct + provider_id: nvidia + provider_model_id: meta/llama3-70b-instruct + model_type: llm +- metadata: {} + model_id: meta-llama/Llama-3.1-8B-Instruct + provider_id: nvidia + provider_model_id: meta/llama-3.1-8b-instruct + model_type: llm +- metadata: {} + model_id: meta-llama/Llama-3.1-70B-Instruct + provider_id: nvidia + provider_model_id: meta/llama-3.1-70b-instruct + model_type: llm +- metadata: {} + model_id: meta-llama/Llama-3.1-405B-Instruct-FP8 + provider_id: nvidia + provider_model_id: meta/llama-3.1-405b-instruct + model_type: llm +- metadata: {} + model_id: meta-llama/Llama-3.2-1B-Instruct + provider_id: nvidia + provider_model_id: meta/llama-3.2-1b-instruct + model_type: llm +- metadata: {} + model_id: meta-llama/Llama-3.2-3B-Instruct + provider_id: nvidia + provider_model_id: meta/llama-3.2-3b-instruct + model_type: llm +- metadata: {} + model_id: meta-llama/Llama-3.2-11B-Vision-Instruct + provider_id: nvidia + provider_model_id: meta/llama-3.2-11b-vision-instruct + model_type: llm +- metadata: {} + model_id: meta-llama/Llama-3.2-90B-Vision-Instruct + provider_id: nvidia + provider_model_id: meta/llama-3.2-90b-vision-instruct + model_type: llm +shields: [] +vector_dbs: [] +datasets: [] +scoring_fns: [] +eval_tasks: [] +tool_groups: +- toolgroup_id: builtin::websearch + provider_id: tavily-search +- toolgroup_id: builtin::rag + provider_id: rag-runtime +- toolgroup_id: builtin::code_interpreter + provider_id: code-interpreter diff --git a/llama_stack/templates/ollama/build.yaml b/llama_stack/templates/ollama/build.yaml index cbd9101cf..0fee6808c 100644 --- a/llama_stack/templates/ollama/build.yaml +++ b/llama_stack/templates/ollama/build.yaml @@ -1,11 +1,10 @@ version: '2' -name: ollama distribution_spec: description: Use (an external) Ollama server for running LLM inference providers: inference: - remote::ollama - memory: + vector_io: - inline::faiss - remote::chromadb - remote::pgvector @@ -28,5 +27,5 @@ distribution_spec: - remote::brave-search - remote::tavily-search - inline::code-interpreter - - inline::memory-runtime + - inline::rag-runtime image_type: conda diff --git a/llama_stack/templates/ollama/ollama.py b/llama_stack/templates/ollama/ollama.py index a9a23c1c4..d14cb3aad 100644 --- a/llama_stack/templates/ollama/ollama.py +++ b/llama_stack/templates/ollama/ollama.py @@ -16,7 +16,7 @@ from llama_stack.distribution.datatypes import ( from llama_stack.providers.inline.inference.sentence_transformers import ( SentenceTransformersInferenceConfig, ) -from llama_stack.providers.inline.memory.faiss.config import FaissImplConfig +from llama_stack.providers.inline.vector_io.faiss.config import FaissImplConfig from llama_stack.providers.remote.inference.ollama import OllamaImplConfig from llama_stack.templates.template import DistributionTemplate, RunConfigSettings @@ -24,7 +24,7 @@ from llama_stack.templates.template import DistributionTemplate, RunConfigSettin def get_distribution_template() -> DistributionTemplate: providers = { "inference": ["remote::ollama"], - "memory": ["inline::faiss", "remote::chromadb", "remote::pgvector"], + "vector_io": ["inline::faiss", "remote::chromadb", "remote::pgvector"], "safety": ["inline::llama-guard"], "agents": ["inline::meta-reference"], "telemetry": ["inline::meta-reference"], @@ -35,7 +35,7 @@ def get_distribution_template() -> DistributionTemplate: "remote::brave-search", "remote::tavily-search", "inline::code-interpreter", - "inline::memory-runtime", + "inline::rag-runtime", ], } name = "ollama" @@ -49,7 +49,7 @@ def get_distribution_template() -> DistributionTemplate: provider_type="inline::sentence-transformers", config=SentenceTransformersInferenceConfig.sample_run_config(), ) - memory_provider = Provider( + vector_io_provider = Provider( provider_id="faiss", provider_type="inline::faiss", config=FaissImplConfig.sample_run_config(f"distributions/{name}"), @@ -77,8 +77,8 @@ def get_distribution_template() -> DistributionTemplate: provider_id="tavily-search", ), ToolGroupInput( - toolgroup_id="builtin::memory", - provider_id="memory-runtime", + toolgroup_id="builtin::rag", + provider_id="rag-runtime", ), ToolGroupInput( toolgroup_id="builtin::code_interpreter", @@ -90,7 +90,7 @@ def get_distribution_template() -> DistributionTemplate: name=name, distro_type="self_hosted", description="Use (an external) Ollama server for running LLM inference", - docker_image=None, + container_image=None, template_path=Path(__file__).parent / "doc_template.md", providers=providers, default_models=[inference_model, safety_model], @@ -98,9 +98,10 @@ def get_distribution_template() -> DistributionTemplate: "run.yaml": RunConfigSettings( provider_overrides={ "inference": [inference_provider, embedding_provider], - "memory": [memory_provider], + "vector_io": [vector_io_provider], }, default_models=[inference_model, embedding_model], + default_tool_groups=default_tool_groups, ), "run-with-safety.yaml": RunConfigSettings( provider_overrides={ @@ -108,7 +109,7 @@ def get_distribution_template() -> DistributionTemplate: inference_provider, embedding_provider, ], - "memory": [memory_provider], + "vector_io": [vector_io_provider], "safety": [ Provider( provider_id="llama-guard", diff --git a/llama_stack/templates/ollama/report.md b/llama_stack/templates/ollama/report.md new file mode 100644 index 000000000..724809a59 --- /dev/null +++ b/llama_stack/templates/ollama/report.md @@ -0,0 +1,44 @@ +# Report for ollama distribution + +## Supported Models +| Model Descriptor | ollama | +|:---|:---| +| Llama-3-8B-Instruct | ❌ | +| Llama-3-70B-Instruct | ❌ | +| Llama3.1-8B-Instruct | ✅ | +| Llama3.1-70B-Instruct | ✅ | +| Llama3.1-405B-Instruct | ✅ | +| Llama3.2-1B-Instruct | ✅ | +| Llama3.2-3B-Instruct | ✅ | +| Llama3.2-11B-Vision-Instruct | ✅ | +| Llama3.2-90B-Vision-Instruct | ✅ | +| Llama3.3-70B-Instruct | ✅ | +| Llama-Guard-3-11B-Vision | ❌ | +| Llama-Guard-3-1B | ✅ | +| Llama-Guard-3-8B | ✅ | +| Llama-Guard-2-8B | ❌ | + +## Inference +| Model | API | Capability | Test | Status | +|:----- |:-----|:-----|:-----|:-----| +| Llama-3.1-8B-Instruct | /chat_completion | streaming | test_text_chat_completion_streaming | ✅ | +| Llama-3.2-11B-Vision-Instruct | /chat_completion | streaming | test_image_chat_completion_streaming | ❌ | +| Llama-3.2-11B-Vision-Instruct | /chat_completion | non_streaming | test_image_chat_completion_non_streaming | ❌ | +| Llama-3.1-8B-Instruct | /chat_completion | non_streaming | test_text_chat_completion_non_streaming | ✅ | +| Llama-3.1-8B-Instruct | /chat_completion | tool_calling | test_text_chat_completion_with_tool_calling_and_streaming | ✅ | +| Llama-3.1-8B-Instruct | /chat_completion | tool_calling | test_text_chat_completion_with_tool_calling_and_non_streaming | ✅ | +| Llama-3.1-8B-Instruct | /completion | streaming | test_text_completion_streaming | ✅ | +| Llama-3.1-8B-Instruct | /completion | non_streaming | test_text_completion_non_streaming | ✅ | +| Llama-3.1-8B-Instruct | /completion | structured_output | test_text_completion_structured_output | ✅ | + +## Vector IO +| API | Capability | Test | Status | +|:-----|:-----|:-----|:-----| +| /retrieve | | test_vector_db_retrieve | ✅ | + +## Agents +| API | Capability | Test | Status | +|:-----|:-----|:-----|:-----| +| /create_agent_turn | rag | test_rag_agent | ✅ | +| /create_agent_turn | custom_tool | test_custom_tool | ✅ | +| /create_agent_turn | code_execution | test_code_interpreter_for_attachments | ✅ | diff --git a/llama_stack/templates/ollama/run-with-safety.yaml b/llama_stack/templates/ollama/run-with-safety.yaml index 0792beddd..5b5c9c253 100644 --- a/llama_stack/templates/ollama/run-with-safety.yaml +++ b/llama_stack/templates/ollama/run-with-safety.yaml @@ -1,16 +1,15 @@ version: '2' image_name: ollama -conda_env: ollama apis: - agents - datasetio - eval - inference -- memory - safety - scoring - telemetry - tool_runtime +- vector_io providers: inference: - provider_id: ollama @@ -20,7 +19,7 @@ providers: - provider_id: sentence-transformers provider_type: inline::sentence-transformers config: {} - memory: + vector_io: - provider_id: faiss provider_type: inline::faiss config: @@ -86,8 +85,8 @@ providers: - provider_id: code-interpreter provider_type: inline::code-interpreter config: {} - - provider_id: memory-runtime - provider_type: inline::memory-runtime + - provider_id: rag-runtime + provider_type: inline::rag-runtime config: {} metadata_store: type: sqlite @@ -111,14 +110,14 @@ shields: provider_id: llama-guard - shield_id: CodeScanner provider_id: code-scanner -memory_banks: [] +vector_dbs: [] datasets: [] scoring_fns: [] eval_tasks: [] tool_groups: - toolgroup_id: builtin::websearch provider_id: tavily-search -- toolgroup_id: builtin::memory - provider_id: memory-runtime +- toolgroup_id: builtin::rag + provider_id: rag-runtime - toolgroup_id: builtin::code_interpreter provider_id: code-interpreter diff --git a/llama_stack/templates/ollama/run.yaml b/llama_stack/templates/ollama/run.yaml index 176465299..3cc1cb2ac 100644 --- a/llama_stack/templates/ollama/run.yaml +++ b/llama_stack/templates/ollama/run.yaml @@ -1,16 +1,15 @@ version: '2' image_name: ollama -conda_env: ollama apis: - agents - datasetio - eval - inference -- memory - safety - scoring - telemetry - tool_runtime +- vector_io providers: inference: - provider_id: ollama @@ -20,7 +19,7 @@ providers: - provider_id: sentence-transformers provider_type: inline::sentence-transformers config: {} - memory: + vector_io: - provider_id: faiss provider_type: inline::faiss config: @@ -83,8 +82,8 @@ providers: - provider_id: code-interpreter provider_type: inline::code-interpreter config: {} - - provider_id: memory-runtime - provider_type: inline::memory-runtime + - provider_id: rag-runtime + provider_type: inline::rag-runtime config: {} metadata_store: type: sqlite @@ -100,8 +99,14 @@ models: provider_id: sentence-transformers model_type: embedding shields: [] -memory_banks: [] +vector_dbs: [] datasets: [] scoring_fns: [] eval_tasks: [] -tool_groups: [] +tool_groups: +- toolgroup_id: builtin::websearch + provider_id: tavily-search +- toolgroup_id: builtin::rag + provider_id: rag-runtime +- toolgroup_id: builtin::code_interpreter + provider_id: code-interpreter diff --git a/llama_stack/templates/remote-vllm/build.yaml b/llama_stack/templates/remote-vllm/build.yaml index 246e53db0..74d9f32d9 100644 --- a/llama_stack/templates/remote-vllm/build.yaml +++ b/llama_stack/templates/remote-vllm/build.yaml @@ -1,11 +1,10 @@ version: '2' -name: remote-vllm distribution_spec: description: Use (an external) vLLM server for running LLM inference providers: inference: - remote::vllm - memory: + vector_io: - inline::faiss - remote::chromadb - remote::pgvector @@ -13,11 +12,21 @@ distribution_spec: - inline::llama-guard agents: - inline::meta-reference + eval: + - inline::meta-reference + datasetio: + - remote::huggingface + - inline::localfs + scoring: + - inline::basic + - inline::llm-as-judge + - inline::braintrust telemetry: - inline::meta-reference tool_runtime: - remote::brave-search - remote::tavily-search - inline::code-interpreter - - inline::memory-runtime + - inline::rag-runtime + - remote::model-context-protocol image_type: conda diff --git a/llama_stack/templates/remote-vllm/run-with-safety.yaml b/llama_stack/templates/remote-vllm/run-with-safety.yaml index 1babd04ac..4a0fa9a85 100644 --- a/llama_stack/templates/remote-vllm/run-with-safety.yaml +++ b/llama_stack/templates/remote-vllm/run-with-safety.yaml @@ -1,13 +1,15 @@ version: '2' image_name: remote-vllm -conda_env: remote-vllm apis: - agents +- datasetio +- eval - inference -- memory - safety +- scoring - telemetry - tool_runtime +- vector_io providers: inference: - provider_id: vllm-inference @@ -25,7 +27,7 @@ providers: - provider_id: sentence-transformers provider_type: inline::sentence-transformers config: {} - memory: + vector_io: - provider_id: faiss provider_type: inline::faiss config: @@ -45,6 +47,28 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/remote-vllm}/agents_store.db + eval: + - provider_id: meta-reference + provider_type: inline::meta-reference + config: {} + datasetio: + - provider_id: huggingface + provider_type: remote::huggingface + config: {} + - provider_id: localfs + provider_type: inline::localfs + config: {} + scoring: + - provider_id: basic + provider_type: inline::basic + config: {} + - provider_id: llm-as-judge + provider_type: inline::llm-as-judge + config: {} + - provider_id: braintrust + provider_type: inline::braintrust + config: + openai_api_key: ${env.OPENAI_API_KEY:} telemetry: - provider_id: meta-reference provider_type: inline::meta-reference @@ -66,8 +90,11 @@ providers: - provider_id: code-interpreter provider_type: inline::code-interpreter config: {} - - provider_id: memory-runtime - provider_type: inline::memory-runtime + - provider_id: rag-runtime + provider_type: inline::rag-runtime + config: {} + - provider_id: model-context-protocol + provider_type: remote::model-context-protocol config: {} metadata_store: type: sqlite @@ -88,14 +115,14 @@ models: model_type: embedding shields: - shield_id: ${env.SAFETY_MODEL} -memory_banks: [] +vector_dbs: [] datasets: [] scoring_fns: [] eval_tasks: [] tool_groups: - toolgroup_id: builtin::websearch provider_id: tavily-search -- toolgroup_id: builtin::memory - provider_id: memory-runtime +- toolgroup_id: builtin::rag + provider_id: rag-runtime - toolgroup_id: builtin::code_interpreter provider_id: code-interpreter diff --git a/llama_stack/templates/remote-vllm/run.yaml b/llama_stack/templates/remote-vllm/run.yaml index a3a571423..9631f94a2 100644 --- a/llama_stack/templates/remote-vllm/run.yaml +++ b/llama_stack/templates/remote-vllm/run.yaml @@ -1,13 +1,15 @@ version: '2' image_name: remote-vllm -conda_env: remote-vllm apis: - agents +- datasetio +- eval - inference -- memory - safety +- scoring - telemetry - tool_runtime +- vector_io providers: inference: - provider_id: vllm-inference @@ -19,7 +21,7 @@ providers: - provider_id: sentence-transformers provider_type: inline::sentence-transformers config: {} - memory: + vector_io: - provider_id: faiss provider_type: inline::faiss config: @@ -39,6 +41,28 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/remote-vllm}/agents_store.db + eval: + - provider_id: meta-reference + provider_type: inline::meta-reference + config: {} + datasetio: + - provider_id: huggingface + provider_type: remote::huggingface + config: {} + - provider_id: localfs + provider_type: inline::localfs + config: {} + scoring: + - provider_id: basic + provider_type: inline::basic + config: {} + - provider_id: llm-as-judge + provider_type: inline::llm-as-judge + config: {} + - provider_id: braintrust + provider_type: inline::braintrust + config: + openai_api_key: ${env.OPENAI_API_KEY:} telemetry: - provider_id: meta-reference provider_type: inline::meta-reference @@ -60,8 +84,11 @@ providers: - provider_id: code-interpreter provider_type: inline::code-interpreter config: {} - - provider_id: memory-runtime - provider_type: inline::memory-runtime + - provider_id: rag-runtime + provider_type: inline::rag-runtime + config: {} + - provider_id: model-context-protocol + provider_type: remote::model-context-protocol config: {} metadata_store: type: sqlite @@ -77,8 +104,14 @@ models: provider_id: sentence-transformers model_type: embedding shields: [] -memory_banks: [] +vector_dbs: [] datasets: [] scoring_fns: [] eval_tasks: [] -tool_groups: [] +tool_groups: +- toolgroup_id: builtin::websearch + provider_id: tavily-search +- toolgroup_id: builtin::rag + provider_id: rag-runtime +- toolgroup_id: builtin::code_interpreter + provider_id: code-interpreter diff --git a/llama_stack/templates/remote-vllm/vllm.py b/llama_stack/templates/remote-vllm/vllm.py index ecaa2cf14..6c835ef86 100644 --- a/llama_stack/templates/remote-vllm/vllm.py +++ b/llama_stack/templates/remote-vllm/vllm.py @@ -16,7 +16,7 @@ from llama_stack.distribution.datatypes import ( from llama_stack.providers.inline.inference.sentence_transformers import ( SentenceTransformersInferenceConfig, ) -from llama_stack.providers.inline.memory.faiss.config import FaissImplConfig +from llama_stack.providers.inline.vector_io.faiss.config import FaissImplConfig from llama_stack.providers.remote.inference.vllm import VLLMInferenceAdapterConfig from llama_stack.templates.template import DistributionTemplate, RunConfigSettings @@ -24,15 +24,19 @@ from llama_stack.templates.template import DistributionTemplate, RunConfigSettin def get_distribution_template() -> DistributionTemplate: providers = { "inference": ["remote::vllm"], - "memory": ["inline::faiss", "remote::chromadb", "remote::pgvector"], + "vector_io": ["inline::faiss", "remote::chromadb", "remote::pgvector"], "safety": ["inline::llama-guard"], "agents": ["inline::meta-reference"], + "eval": ["inline::meta-reference"], + "datasetio": ["remote::huggingface", "inline::localfs"], + "scoring": ["inline::basic", "inline::llm-as-judge", "inline::braintrust"], "telemetry": ["inline::meta-reference"], "tool_runtime": [ "remote::brave-search", "remote::tavily-search", "inline::code-interpreter", - "inline::memory-runtime", + "inline::rag-runtime", + "remote::model-context-protocol", ], } name = "remote-vllm" @@ -48,7 +52,7 @@ def get_distribution_template() -> DistributionTemplate: provider_type="inline::sentence-transformers", config=SentenceTransformersInferenceConfig.sample_run_config(), ) - memory_provider = Provider( + vector_io_provider = Provider( provider_id="faiss", provider_type="inline::faiss", config=FaissImplConfig.sample_run_config(f"distributions/{name}"), @@ -76,8 +80,8 @@ def get_distribution_template() -> DistributionTemplate: provider_id="tavily-search", ), ToolGroupInput( - toolgroup_id="builtin::memory", - provider_id="memory-runtime", + toolgroup_id="builtin::rag", + provider_id="rag-runtime", ), ToolGroupInput( toolgroup_id="builtin::code_interpreter", @@ -96,9 +100,10 @@ def get_distribution_template() -> DistributionTemplate: "run.yaml": RunConfigSettings( provider_overrides={ "inference": [inference_provider, embedding_provider], - "memory": [memory_provider], + "vector_io": [vector_io_provider], }, default_models=[inference_model, embedding_model], + default_tool_groups=default_tool_groups, ), "run-with-safety.yaml": RunConfigSettings( provider_overrides={ @@ -113,7 +118,7 @@ def get_distribution_template() -> DistributionTemplate: ), embedding_provider, ], - "memory": [memory_provider], + "vector_io": [vector_io_provider], }, default_models=[ inference_model, @@ -134,7 +139,7 @@ def get_distribution_template() -> DistributionTemplate: "Inference model loaded into the vLLM server", ), "VLLM_URL": ( - "http://host.docker.internal:5100}/v1", + "http://host.docker.internal:5100/v1", "URL of the vLLM server with the main inference model", ), "MAX_TOKENS": ( diff --git a/llama_stack/templates/template.py b/llama_stack/templates/template.py index 5bb88c821..78f57b795 100644 --- a/llama_stack/templates/template.py +++ b/llama_stack/templates/template.py @@ -37,7 +37,7 @@ class RunConfigSettings(BaseModel): self, name: str, providers: Dict[str, List[str]], - docker_image: Optional[str] = None, + container_image: Optional[str] = None, ) -> StackRunConfig: provider_registry = get_provider_registry() @@ -83,8 +83,7 @@ class RunConfigSettings(BaseModel): return StackRunConfig( image_name=name, - docker_image=docker_image, - conda_env=name, + container_image=container_image, apis=apis, providers=provider_configs, metadata_store=SqliteKVStoreConfig.sample_run_config( @@ -113,7 +112,7 @@ class DistributionTemplate(BaseModel): # Optional configuration run_config_env_vars: Optional[Dict[str, Tuple[str, str]]] = None - docker_image: Optional[str] = None + container_image: Optional[str] = None default_models: Optional[List[ModelInput]] = None @@ -122,7 +121,7 @@ class DistributionTemplate(BaseModel): name=self.name, distribution_spec=DistributionSpec( description=self.description, - docker_image=self.docker_image, + container_image=self.container_image, providers=self.providers, ), image_type="conda", # default to conda, can be overridden @@ -170,7 +169,7 @@ class DistributionTemplate(BaseModel): for yaml_pth, settings in self.run_configs.items(): run_config = settings.run_config( - self.name, self.providers, self.docker_image + self.name, self.providers, self.container_image ) with open(yaml_output_dir / yaml_pth, "w") as f: yaml.safe_dump( diff --git a/llama_stack/templates/tgi/build.yaml b/llama_stack/templates/tgi/build.yaml index 399d4a616..8bc628158 100644 --- a/llama_stack/templates/tgi/build.yaml +++ b/llama_stack/templates/tgi/build.yaml @@ -1,11 +1,10 @@ version: '2' -name: tgi distribution_spec: description: Use (an external) TGI server for running LLM inference providers: inference: - remote::tgi - memory: + vector_io: - inline::faiss - remote::chromadb - remote::pgvector @@ -28,5 +27,6 @@ distribution_spec: - remote::brave-search - remote::tavily-search - inline::code-interpreter - - inline::memory-runtime + - inline::rag-runtime + - remote::model-context-protocol image_type: conda diff --git a/llama_stack/templates/tgi/report.md b/llama_stack/templates/tgi/report.md new file mode 100644 index 000000000..b0f5d88a2 --- /dev/null +++ b/llama_stack/templates/tgi/report.md @@ -0,0 +1,44 @@ +# Report for tgi distribution + +## Supported Models +| Model Descriptor | tgi | +|:---|:---| +| Llama-3-8B-Instruct | ✅ | +| Llama-3-70B-Instruct | ✅ | +| Llama3.1-8B-Instruct | ✅ | +| Llama3.1-70B-Instruct | ✅ | +| Llama3.1-405B-Instruct | ✅ | +| Llama3.2-1B-Instruct | ✅ | +| Llama3.2-3B-Instruct | ✅ | +| Llama3.2-11B-Vision-Instruct | ✅ | +| Llama3.2-90B-Vision-Instruct | ✅ | +| Llama3.3-70B-Instruct | ✅ | +| Llama-Guard-3-11B-Vision | ✅ | +| Llama-Guard-3-1B | ✅ | +| Llama-Guard-3-8B | ✅ | +| Llama-Guard-2-8B | ✅ | + +## Inference +| Model | API | Capability | Test | Status | +|:----- |:-----|:-----|:-----|:-----| +| Llama-3.1-8B-Instruct | /chat_completion | streaming | test_text_chat_completion_streaming | ✅ | +| Llama-3.2-11B-Vision-Instruct | /chat_completion | streaming | test_image_chat_completion_streaming | ❌ | +| Llama-3.2-11B-Vision-Instruct | /chat_completion | non_streaming | test_image_chat_completion_non_streaming | ❌ | +| Llama-3.1-8B-Instruct | /chat_completion | non_streaming | test_text_chat_completion_non_streaming | ✅ | +| Llama-3.1-8B-Instruct | /chat_completion | tool_calling | test_text_chat_completion_with_tool_calling_and_streaming | ✅ | +| Llama-3.1-8B-Instruct | /chat_completion | tool_calling | test_text_chat_completion_with_tool_calling_and_non_streaming | ✅ | +| Llama-3.1-8B-Instruct | /completion | streaming | test_text_completion_streaming | ✅ | +| Llama-3.1-8B-Instruct | /completion | non_streaming | test_text_completion_non_streaming | ✅ | +| Llama-3.1-8B-Instruct | /completion | structured_output | test_text_completion_structured_output | ✅ | + +## Vector IO +| API | Capability | Test | Status | +|:-----|:-----|:-----|:-----| +| /retrieve | | test_vector_db_retrieve | ✅ | + +## Agents +| API | Capability | Test | Status | +|:-----|:-----|:-----|:-----| +| /create_agent_turn | rag | test_rag_agent | ✅ | +| /create_agent_turn | custom_tool | test_custom_tool | ✅ | +| /create_agent_turn | code_execution | test_code_interpreter_for_attachments | ✅ | diff --git a/llama_stack/templates/tgi/run-with-safety.yaml b/llama_stack/templates/tgi/run-with-safety.yaml index 4134101f6..503505c32 100644 --- a/llama_stack/templates/tgi/run-with-safety.yaml +++ b/llama_stack/templates/tgi/run-with-safety.yaml @@ -1,16 +1,15 @@ version: '2' image_name: tgi -conda_env: tgi apis: - agents - datasetio - eval - inference -- memory - safety - scoring - telemetry - tool_runtime +- vector_io providers: inference: - provider_id: tgi-inference @@ -21,7 +20,7 @@ providers: provider_type: remote::tgi config: url: ${env.TGI_SAFETY_URL} - memory: + vector_io: - provider_id: faiss provider_type: inline::faiss config: @@ -84,8 +83,11 @@ providers: - provider_id: code-interpreter provider_type: inline::code-interpreter config: {} - - provider_id: memory-runtime - provider_type: inline::memory-runtime + - provider_id: rag-runtime + provider_type: inline::rag-runtime + config: {} + - provider_id: model-context-protocol + provider_type: remote::model-context-protocol config: {} metadata_store: type: sqlite @@ -101,14 +103,14 @@ models: model_type: llm shields: - shield_id: ${env.SAFETY_MODEL} -memory_banks: [] +vector_dbs: [] datasets: [] scoring_fns: [] eval_tasks: [] tool_groups: - toolgroup_id: builtin::websearch provider_id: tavily-search -- toolgroup_id: builtin::memory - provider_id: memory-runtime +- toolgroup_id: builtin::rag + provider_id: rag-runtime - toolgroup_id: builtin::code_interpreter provider_id: code-interpreter diff --git a/llama_stack/templates/tgi/run.yaml b/llama_stack/templates/tgi/run.yaml index b0b78e33b..f1953c513 100644 --- a/llama_stack/templates/tgi/run.yaml +++ b/llama_stack/templates/tgi/run.yaml @@ -1,16 +1,15 @@ version: '2' image_name: tgi -conda_env: tgi apis: - agents - datasetio - eval - inference -- memory - safety - scoring - telemetry - tool_runtime +- vector_io providers: inference: - provider_id: tgi-inference @@ -20,7 +19,7 @@ providers: - provider_id: sentence-transformers provider_type: inline::sentence-transformers config: {} - memory: + vector_io: - provider_id: faiss provider_type: inline::faiss config: @@ -83,8 +82,11 @@ providers: - provider_id: code-interpreter provider_type: inline::code-interpreter config: {} - - provider_id: memory-runtime - provider_type: inline::memory-runtime + - provider_id: rag-runtime + provider_type: inline::rag-runtime + config: {} + - provider_id: model-context-protocol + provider_type: remote::model-context-protocol config: {} metadata_store: type: sqlite @@ -100,8 +102,14 @@ models: provider_id: sentence-transformers model_type: embedding shields: [] -memory_banks: [] +vector_dbs: [] datasets: [] scoring_fns: [] eval_tasks: [] -tool_groups: [] +tool_groups: +- toolgroup_id: builtin::websearch + provider_id: tavily-search +- toolgroup_id: builtin::rag + provider_id: rag-runtime +- toolgroup_id: builtin::code_interpreter + provider_id: code-interpreter diff --git a/llama_stack/templates/tgi/tgi.py b/llama_stack/templates/tgi/tgi.py index 37ed2751b..e49c98d72 100644 --- a/llama_stack/templates/tgi/tgi.py +++ b/llama_stack/templates/tgi/tgi.py @@ -16,7 +16,7 @@ from llama_stack.distribution.datatypes import ( from llama_stack.providers.inline.inference.sentence_transformers import ( SentenceTransformersInferenceConfig, ) -from llama_stack.providers.inline.memory.faiss.config import FaissImplConfig +from llama_stack.providers.inline.vector_io.faiss.config import FaissImplConfig from llama_stack.providers.remote.inference.tgi import TGIImplConfig from llama_stack.templates.template import DistributionTemplate, RunConfigSettings @@ -24,7 +24,7 @@ from llama_stack.templates.template import DistributionTemplate, RunConfigSettin def get_distribution_template() -> DistributionTemplate: providers = { "inference": ["remote::tgi"], - "memory": ["inline::faiss", "remote::chromadb", "remote::pgvector"], + "vector_io": ["inline::faiss", "remote::chromadb", "remote::pgvector"], "safety": ["inline::llama-guard"], "agents": ["inline::meta-reference"], "telemetry": ["inline::meta-reference"], @@ -35,7 +35,8 @@ def get_distribution_template() -> DistributionTemplate: "remote::brave-search", "remote::tavily-search", "inline::code-interpreter", - "inline::memory-runtime", + "inline::rag-runtime", + "remote::model-context-protocol", ], } name = "tgi" @@ -51,7 +52,7 @@ def get_distribution_template() -> DistributionTemplate: provider_type="inline::sentence-transformers", config=SentenceTransformersInferenceConfig.sample_run_config(), ) - memory_provider = Provider( + vector_io_provider = Provider( provider_id="faiss", provider_type="inline::faiss", config=FaissImplConfig.sample_run_config(f"distributions/{name}"), @@ -79,8 +80,8 @@ def get_distribution_template() -> DistributionTemplate: provider_id="tavily-search", ), ToolGroupInput( - toolgroup_id="builtin::memory", - provider_id="memory-runtime", + toolgroup_id="builtin::rag", + provider_id="rag-runtime", ), ToolGroupInput( toolgroup_id="builtin::code_interpreter", @@ -92,7 +93,7 @@ def get_distribution_template() -> DistributionTemplate: name=name, distro_type="self_hosted", description="Use (an external) TGI server for running LLM inference", - docker_image=None, + container_image=None, template_path=Path(__file__).parent / "doc_template.md", providers=providers, default_models=[inference_model, safety_model], @@ -100,9 +101,10 @@ def get_distribution_template() -> DistributionTemplate: "run.yaml": RunConfigSettings( provider_overrides={ "inference": [inference_provider, embedding_provider], - "memory": [memory_provider], + "vector_io": [vector_io_provider], }, default_models=[inference_model, embedding_model], + default_tool_groups=default_tool_groups, ), "run-with-safety.yaml": RunConfigSettings( provider_overrides={ @@ -116,7 +118,7 @@ def get_distribution_template() -> DistributionTemplate: ), ), ], - "memory": [memory_provider], + "vector_io": [vector_io_provider], }, default_models=[ inference_model, diff --git a/llama_stack/templates/together/build.yaml b/llama_stack/templates/together/build.yaml index 96f9f758e..90ee5bcee 100644 --- a/llama_stack/templates/together/build.yaml +++ b/llama_stack/templates/together/build.yaml @@ -1,11 +1,10 @@ version: '2' -name: together distribution_spec: description: Use Together.AI for running LLM inference providers: inference: - remote::together - memory: + vector_io: - inline::faiss - remote::chromadb - remote::pgvector @@ -28,5 +27,6 @@ distribution_spec: - remote::brave-search - remote::tavily-search - inline::code-interpreter - - inline::memory-runtime + - inline::rag-runtime + - remote::model-context-protocol image_type: conda diff --git a/llama_stack/templates/together/report.md b/llama_stack/templates/together/report.md new file mode 100644 index 000000000..b5339c640 --- /dev/null +++ b/llama_stack/templates/together/report.md @@ -0,0 +1,44 @@ +# Report for together distribution + +## Supported Models +| Model Descriptor | together | +|:---|:---| +| Llama-3-8B-Instruct | ❌ | +| Llama-3-70B-Instruct | ❌ | +| Llama3.1-8B-Instruct | ✅ | +| Llama3.1-70B-Instruct | ✅ | +| Llama3.1-405B-Instruct | ✅ | +| Llama3.2-1B-Instruct | ❌ | +| Llama3.2-3B-Instruct | ✅ | +| Llama3.2-11B-Vision-Instruct | ✅ | +| Llama3.2-90B-Vision-Instruct | ✅ | +| Llama3.3-70B-Instruct | ✅ | +| Llama-Guard-3-11B-Vision | ✅ | +| Llama-Guard-3-1B | ❌ | +| Llama-Guard-3-8B | ✅ | +| Llama-Guard-2-8B | ❌ | + +## Inference +| Model | API | Capability | Test | Status | +|:----- |:-----|:-----|:-----|:-----| +| Llama-3.1-8B-Instruct | /chat_completion | streaming | test_text_chat_completion_streaming | ✅ | +| Llama-3.2-11B-Vision-Instruct | /chat_completion | streaming | test_image_chat_completion_streaming | ✅ | +| Llama-3.2-11B-Vision-Instruct | /chat_completion | non_streaming | test_image_chat_completion_non_streaming | ✅ | +| Llama-3.1-8B-Instruct | /chat_completion | non_streaming | test_text_chat_completion_non_streaming | ✅ | +| Llama-3.1-8B-Instruct | /chat_completion | tool_calling | test_text_chat_completion_with_tool_calling_and_streaming | ✅ | +| Llama-3.1-8B-Instruct | /chat_completion | tool_calling | test_text_chat_completion_with_tool_calling_and_non_streaming | ✅ | +| Llama-3.1-8B-Instruct | /completion | streaming | test_text_completion_streaming | ✅ | +| Llama-3.1-8B-Instruct | /completion | non_streaming | test_text_completion_non_streaming | ✅ | +| Llama-3.1-8B-Instruct | /completion | structured_output | test_text_completion_structured_output | ✅ | + +## Vector IO +| API | Capability | Test | Status | +|:-----|:-----|:-----|:-----| +| /retrieve | | test_vector_db_retrieve | ✅ | + +## Agents +| API | Capability | Test | Status | +|:-----|:-----|:-----|:-----| +| /create_agent_turn | rag | test_rag_agent | ✅ | +| /create_agent_turn | custom_tool | test_custom_tool | ✅ | +| /create_agent_turn | code_execution | test_code_interpreter_for_attachments | ✅ | diff --git a/llama_stack/templates/together/run-with-safety.yaml b/llama_stack/templates/together/run-with-safety.yaml index c415b0ec0..ec351108e 100644 --- a/llama_stack/templates/together/run-with-safety.yaml +++ b/llama_stack/templates/together/run-with-safety.yaml @@ -1,16 +1,15 @@ version: '2' image_name: together -conda_env: together apis: - agents - datasetio - eval - inference -- memory - safety - scoring - telemetry - tool_runtime +- vector_io providers: inference: - provider_id: together @@ -21,7 +20,7 @@ providers: - provider_id: sentence-transformers provider_type: inline::sentence-transformers config: {} - memory: + vector_io: - provider_id: faiss provider_type: inline::faiss config: @@ -90,8 +89,11 @@ providers: - provider_id: code-interpreter provider_type: inline::code-interpreter config: {} - - provider_id: memory-runtime - provider_type: inline::memory-runtime + - provider_id: rag-runtime + provider_type: inline::rag-runtime + config: {} + - provider_id: model-context-protocol + provider_type: remote::model-context-protocol config: {} metadata_store: type: sqlite @@ -154,14 +156,14 @@ shields: provider_id: llama-guard-vision - shield_id: CodeScanner provider_id: code-scanner -memory_banks: [] +vector_dbs: [] datasets: [] scoring_fns: [] eval_tasks: [] tool_groups: - toolgroup_id: builtin::websearch provider_id: tavily-search -- toolgroup_id: builtin::memory - provider_id: memory-runtime +- toolgroup_id: builtin::rag + provider_id: rag-runtime - toolgroup_id: builtin::code_interpreter provider_id: code-interpreter diff --git a/llama_stack/templates/together/run.yaml b/llama_stack/templates/together/run.yaml index ed65ded57..c2afd98e9 100644 --- a/llama_stack/templates/together/run.yaml +++ b/llama_stack/templates/together/run.yaml @@ -1,16 +1,15 @@ version: '2' image_name: together -conda_env: together apis: - agents - datasetio - eval - inference -- memory - safety - scoring - telemetry - tool_runtime +- vector_io providers: inference: - provider_id: together @@ -21,7 +20,7 @@ providers: - provider_id: sentence-transformers provider_type: inline::sentence-transformers config: {} - memory: + vector_io: - provider_id: faiss provider_type: inline::faiss config: @@ -84,8 +83,11 @@ providers: - provider_id: code-interpreter provider_type: inline::code-interpreter config: {} - - provider_id: memory-runtime - provider_type: inline::memory-runtime + - provider_id: rag-runtime + provider_type: inline::rag-runtime + config: {} + - provider_id: model-context-protocol + provider_type: remote::model-context-protocol config: {} metadata_store: type: sqlite @@ -143,14 +145,14 @@ models: model_type: embedding shields: - shield_id: meta-llama/Llama-Guard-3-8B -memory_banks: [] +vector_dbs: [] datasets: [] scoring_fns: [] eval_tasks: [] tool_groups: - toolgroup_id: builtin::websearch provider_id: tavily-search -- toolgroup_id: builtin::memory - provider_id: memory-runtime +- toolgroup_id: builtin::rag + provider_id: rag-runtime - toolgroup_id: builtin::code_interpreter provider_id: code-interpreter diff --git a/llama_stack/templates/together/together.py b/llama_stack/templates/together/together.py index b51918a6c..5e9520433 100644 --- a/llama_stack/templates/together/together.py +++ b/llama_stack/templates/together/together.py @@ -18,7 +18,7 @@ from llama_stack.distribution.datatypes import ( from llama_stack.providers.inline.inference.sentence_transformers import ( SentenceTransformersInferenceConfig, ) -from llama_stack.providers.inline.memory.faiss.config import FaissImplConfig +from llama_stack.providers.inline.vector_io.faiss.config import FaissImplConfig from llama_stack.providers.remote.inference.together import TogetherImplConfig from llama_stack.providers.remote.inference.together.together import MODEL_ALIASES from llama_stack.templates.template import DistributionTemplate, RunConfigSettings @@ -27,7 +27,7 @@ from llama_stack.templates.template import DistributionTemplate, RunConfigSettin def get_distribution_template() -> DistributionTemplate: providers = { "inference": ["remote::together"], - "memory": ["inline::faiss", "remote::chromadb", "remote::pgvector"], + "vector_io": ["inline::faiss", "remote::chromadb", "remote::pgvector"], "safety": ["inline::llama-guard"], "agents": ["inline::meta-reference"], "telemetry": ["inline::meta-reference"], @@ -38,7 +38,8 @@ def get_distribution_template() -> DistributionTemplate: "remote::brave-search", "remote::tavily-search", "inline::code-interpreter", - "inline::memory-runtime", + "inline::rag-runtime", + "remote::model-context-protocol", ], } name = "together" @@ -47,7 +48,7 @@ def get_distribution_template() -> DistributionTemplate: provider_type="remote::together", config=TogetherImplConfig.sample_run_config(), ) - memory_provider = Provider( + vector_io_provider = Provider( provider_id="faiss", provider_type="inline::faiss", config=FaissImplConfig.sample_run_config(f"distributions/{name}"), @@ -75,8 +76,8 @@ def get_distribution_template() -> DistributionTemplate: provider_id="tavily-search", ), ToolGroupInput( - toolgroup_id="builtin::memory", - provider_id="memory-runtime", + toolgroup_id="builtin::rag", + provider_id="rag-runtime", ), ToolGroupInput( toolgroup_id="builtin::code_interpreter", @@ -96,7 +97,7 @@ def get_distribution_template() -> DistributionTemplate: name=name, distro_type="self_hosted", description="Use Together.AI for running LLM inference", - docker_image=None, + container_image=None, template_path=Path(__file__).parent / "doc_template.md", providers=providers, default_models=default_models, @@ -104,7 +105,7 @@ def get_distribution_template() -> DistributionTemplate: "run.yaml": RunConfigSettings( provider_overrides={ "inference": [inference_provider, embedding_provider], - "memory": [memory_provider], + "vector_io": [vector_io_provider], }, default_models=default_models + [embedding_model], default_tool_groups=default_tool_groups, @@ -116,7 +117,7 @@ def get_distribution_template() -> DistributionTemplate: inference_provider, embedding_provider, ], - "memory": [memory_provider], + "vector_io": [vector_io_provider], "safety": [ Provider( provider_id="llama-guard", diff --git a/llama_stack/templates/vllm-gpu/build.yaml b/llama_stack/templates/vllm-gpu/build.yaml index 959f91d3e..d24046613 100644 --- a/llama_stack/templates/vllm-gpu/build.yaml +++ b/llama_stack/templates/vllm-gpu/build.yaml @@ -1,11 +1,10 @@ version: '2' -name: vllm-gpu distribution_spec: description: Use a built-in vLLM engine for running LLM inference providers: inference: - inline::vllm - memory: + vector_io: - inline::faiss - remote::chromadb - remote::pgvector @@ -28,5 +27,6 @@ distribution_spec: - remote::brave-search - remote::tavily-search - inline::code-interpreter - - inline::memory-runtime + - inline::rag-runtime + - remote::model-context-protocol image_type: conda diff --git a/llama_stack/templates/vllm-gpu/run.yaml b/llama_stack/templates/vllm-gpu/run.yaml index 48ec57cfb..165e4d51d 100644 --- a/llama_stack/templates/vllm-gpu/run.yaml +++ b/llama_stack/templates/vllm-gpu/run.yaml @@ -1,16 +1,15 @@ version: '2' image_name: vllm-gpu -conda_env: vllm-gpu apis: - agents - datasetio - eval - inference -- memory - safety - scoring - telemetry - tool_runtime +- vector_io providers: inference: - provider_id: vllm @@ -24,7 +23,7 @@ providers: - provider_id: sentence-transformers provider_type: inline::sentence-transformers config: {} - memory: + vector_io: - provider_id: faiss provider_type: inline::faiss config: @@ -87,8 +86,11 @@ providers: - provider_id: code-interpreter provider_type: inline::code-interpreter config: {} - - provider_id: memory-runtime - provider_type: inline::memory-runtime + - provider_id: rag-runtime + provider_type: inline::rag-runtime + config: {} + - provider_id: model-context-protocol + provider_type: remote::model-context-protocol config: {} metadata_store: type: sqlite @@ -104,14 +106,14 @@ models: provider_id: sentence-transformers model_type: embedding shields: [] -memory_banks: [] +vector_dbs: [] datasets: [] scoring_fns: [] eval_tasks: [] tool_groups: - toolgroup_id: builtin::websearch provider_id: tavily-search -- toolgroup_id: builtin::memory - provider_id: memory-runtime +- toolgroup_id: builtin::rag + provider_id: rag-runtime - toolgroup_id: builtin::code_interpreter provider_id: code-interpreter diff --git a/llama_stack/templates/vllm-gpu/vllm.py b/llama_stack/templates/vllm-gpu/vllm.py index dd80c15dc..54ebd2d41 100644 --- a/llama_stack/templates/vllm-gpu/vllm.py +++ b/llama_stack/templates/vllm-gpu/vllm.py @@ -10,7 +10,7 @@ from llama_stack.providers.inline.inference.sentence_transformers import ( SentenceTransformersInferenceConfig, ) from llama_stack.providers.inline.inference.vllm import VLLMConfig -from llama_stack.providers.inline.memory.faiss.config import FaissImplConfig +from llama_stack.providers.inline.vector_io.faiss.config import FaissImplConfig from llama_stack.templates.template import ( DistributionTemplate, RunConfigSettings, @@ -21,7 +21,7 @@ from llama_stack.templates.template import ( def get_distribution_template() -> DistributionTemplate: providers = { "inference": ["inline::vllm"], - "memory": ["inline::faiss", "remote::chromadb", "remote::pgvector"], + "vector_io": ["inline::faiss", "remote::chromadb", "remote::pgvector"], "safety": ["inline::llama-guard"], "agents": ["inline::meta-reference"], "telemetry": ["inline::meta-reference"], @@ -32,7 +32,8 @@ def get_distribution_template() -> DistributionTemplate: "remote::brave-search", "remote::tavily-search", "inline::code-interpreter", - "inline::memory-runtime", + "inline::rag-runtime", + "remote::model-context-protocol", ], } @@ -42,7 +43,7 @@ def get_distribution_template() -> DistributionTemplate: provider_type="inline::vllm", config=VLLMConfig.sample_run_config(), ) - memory_provider = Provider( + vector_io_provider = Provider( provider_id="faiss", provider_type="inline::faiss", config=FaissImplConfig.sample_run_config(f"distributions/{name}"), @@ -71,8 +72,8 @@ def get_distribution_template() -> DistributionTemplate: provider_id="tavily-search", ), ToolGroupInput( - toolgroup_id="builtin::memory", - provider_id="memory-runtime", + toolgroup_id="builtin::rag", + provider_id="rag-runtime", ), ToolGroupInput( toolgroup_id="builtin::code_interpreter", @@ -84,7 +85,7 @@ def get_distribution_template() -> DistributionTemplate: name=name, distro_type="self_hosted", description="Use a built-in vLLM engine for running LLM inference", - docker_image=None, + container_image=None, template_path=None, providers=providers, default_models=[inference_model], @@ -92,7 +93,7 @@ def get_distribution_template() -> DistributionTemplate: "run.yaml": RunConfigSettings( provider_overrides={ "inference": [inference_provider, embedding_provider], - "memory": [memory_provider], + "vector_io": [vector_io_provider], }, default_models=[inference_model, embedding_model], default_tool_groups=default_tool_groups, diff --git a/tests/client-sdk/README.md b/tests/client-sdk/README.md new file mode 100644 index 000000000..2edf6d3c8 --- /dev/null +++ b/tests/client-sdk/README.md @@ -0,0 +1,21 @@ +# Llama Stack Integration Tests +You can run llama stack integration tests on either a Llama Stack Library or a Llama Stack endpoint. + +To test on a Llama Stack library with certain configuration, run +```bash +LLAMA_STACK_CONFIG=./llama_stack/templates/cerebras/run.yaml +pytest -s -v tests/client-sdk/inference/test_inference.py +``` + +To test on a Llama Stack endpoint, run +```bash +LLAMA_STACK_BASE_URL=http//localhost:8089 +pytest -s -v tests/client-sdk/inference/test_inference.py +``` + + +## Common options +Depending on the API, there are custom options enabled +- For tests in `inference/` and `agents/, we support `--inference-model` (to be used in text inference tests) and `--vision-inference-model` (only used in image inference tests) overrides +- For tests in `vector_io/`, we support `--embedding-model` override +- For tests in `safety/`, we support `--safety-shield` override diff --git a/tests/client-sdk/agents/test_agents.py b/tests/client-sdk/agents/test_agents.py index 0c16b6225..c6be91232 100644 --- a/tests/client-sdk/agents/test_agents.py +++ b/tests/client-sdk/agents/test_agents.py @@ -80,26 +80,21 @@ class TestClientTool(ClientTool): @pytest.fixture(scope="session") -def agent_config(llama_stack_client): - available_models = [ - model.identifier - for model in llama_stack_client.models.list() - if model.identifier.startswith("meta-llama") and "405" not in model.identifier - ] - model_id = available_models[0] - print(f"Using model: {model_id}") +def agent_config(llama_stack_client, text_model_id): available_shields = [ shield.identifier for shield in llama_stack_client.shields.list() ] available_shields = available_shields[:1] print(f"Using shield: {available_shields}") agent_config = AgentConfig( - model=model_id, + model=text_model_id, instructions="You are a helpful assistant", sampling_params={ - "strategy": "greedy", - "temperature": 1.0, - "top_p": 0.9, + "strategy": { + "type": "top_p", + "temperature": 1.0, + "top_p": 0.9, + }, }, toolgroups=[], tool_choice="auto", @@ -175,7 +170,8 @@ def test_builtin_tool_web_search(llama_stack_client, agent_config): assert "tool_execution>" in logs_str assert "Tool:brave_search Response:" in logs_str assert "mark zuckerberg" in logs_str.lower() - assert "No Violation" in logs_str + if len(agent_config["output_shields"]) > 0: + assert "No Violation" in logs_str def test_builtin_tool_code_execution(llama_stack_client, agent_config): @@ -204,18 +200,16 @@ def test_builtin_tool_code_execution(llama_stack_client, agent_config): assert "Tool:code_interpreter Response" in logs_str -def test_code_execution(llama_stack_client): - agent_config = AgentConfig( - model="meta-llama/Llama-3.1-8B-Instruct", - instructions="You are a helpful assistant", - toolgroups=[ +# This test must be run in an environment where `bwrap` is available. If you are running against a +# server, this means the _server_ must have `bwrap` available. If you are using library client, then +# you must have `bwrap` available in test's environment. +def test_code_interpreter_for_attachments(llama_stack_client, agent_config): + agent_config = { + **agent_config, + "toolgroups": [ "builtin::code_interpreter", ], - tool_choice="required", - input_shields=[], - output_shields=[], - enable_session_persistence=False, - ) + } codex_agent = Agent(llama_stack_client, agent_config) session_id = codex_agent.create_session("test-session") @@ -249,10 +243,8 @@ def test_custom_tool(llama_stack_client, agent_config): client_tool = TestClientTool() agent_config = { **agent_config, - "model": "meta-llama/Llama-3.2-3B-Instruct", "toolgroups": ["builtin::websearch"], "client_tools": [client_tool.get_tool_definition()], - "tool_prompt_format": "python_list", } agent = Agent(llama_stack_client, agent_config, client_tools=(client_tool,)) @@ -285,27 +277,24 @@ def test_rag_agent(llama_stack_client, agent_config): ) for i, url in enumerate(urls) ] - memory_bank_id = "test-memory-bank" - llama_stack_client.memory_banks.register( - memory_bank_id=memory_bank_id, - params={ - "memory_bank_type": "vector", - "embedding_model": "all-MiniLM-L6-v2", - "chunk_size_in_tokens": 512, - "overlap_size_in_tokens": 64, - }, + vector_db_id = "test-vector-db" + llama_stack_client.vector_dbs.register( + vector_db_id=vector_db_id, + embedding_model="all-MiniLM-L6-v2", + embedding_dimension=384, ) - llama_stack_client.memory.insert( - bank_id=memory_bank_id, + llama_stack_client.tool_runtime.rag_tool.insert( documents=documents, + vector_db_id=vector_db_id, + chunk_size_in_tokens=512, ) agent_config = { **agent_config, "toolgroups": [ dict( - name="builtin::memory", + name="builtin::rag", args={ - "memory_bank_ids": [memory_bank_id], + "vector_db_ids": [vector_db_id], }, ) ], @@ -323,4 +312,4 @@ def test_rag_agent(llama_stack_client, agent_config): ) logs = [str(log) for log in EventLogger().log(response) if log is not None] logs_str = "".join(logs) - assert "Tool:query_memory" in logs_str + assert "Tool:query_from_memory" in logs_str diff --git a/tests/client-sdk/conftest.py b/tests/client-sdk/conftest.py index b40d54ee5..779c10e21 100644 --- a/tests/client-sdk/conftest.py +++ b/tests/client-sdk/conftest.py @@ -10,11 +10,39 @@ import pytest from llama_stack import LlamaStackAsLibraryClient from llama_stack.providers.tests.env import get_env_or_fail from llama_stack_client import LlamaStackClient +from report import Report def pytest_configure(config): config.option.tbstyle = "short" config.option.disable_warnings = True + if config.getoption("--report"): + config.pluginmanager.register(Report()) + + +TEXT_MODEL = "meta-llama/Llama-3.1-8B-Instruct" +VISION_MODEL = "meta-llama/Llama-3.2-11B-Vision-Instruct" + + +def pytest_addoption(parser): + parser.addoption( + "--report", + default=False, + action="store_true", + help="Knob to determine if we should generate report, e.g. --output=True", + ) + parser.addoption( + "--inference-model", + action="store", + default=TEXT_MODEL, + help="Specify the inference model to use for testing", + ) + parser.addoption( + "--vision-inference-model", + action="store", + default=VISION_MODEL, + help="Specify the vision inference model to use for testing", + ) @pytest.fixture(scope="session") @@ -36,7 +64,9 @@ def llama_stack_client(provider_data): provider_data=provider_data, skip_logger_removal=True, ) - client.initialize() + if not client.initialize(): + raise RuntimeError("Initialization failed") + elif os.environ.get("LLAMA_STACK_BASE_URL"): client = LlamaStackClient( base_url=get_env_or_fail("LLAMA_STACK_BASE_URL"), @@ -45,3 +75,18 @@ def llama_stack_client(provider_data): else: raise ValueError("LLAMA_STACK_CONFIG or LLAMA_STACK_BASE_URL must be set") return client + + +def pytest_generate_tests(metafunc): + if "text_model_id" in metafunc.fixturenames: + metafunc.parametrize( + "text_model_id", + [metafunc.config.getoption("--inference-model")], + scope="session", + ) + if "vision_model_id" in metafunc.fixturenames: + metafunc.parametrize( + "vision_model_id", + [metafunc.config.getoption("--vision-inference-model")], + scope="session", + ) diff --git a/tests/client-sdk/inference/dog.png b/tests/client-sdk/inference/dog.png new file mode 100644 index 000000000..2d502e606 Binary files /dev/null and b/tests/client-sdk/inference/dog.png differ diff --git a/tests/client-sdk/inference/test_inference.py b/tests/client-sdk/inference/test_inference.py index a50dba3a0..8ca11521c 100644 --- a/tests/client-sdk/inference/test_inference.py +++ b/tests/client-sdk/inference/test_inference.py @@ -4,13 +4,14 @@ # This source code is licensed under the terms described in the LICENSE file in # the root directory of this source tree. -import pytest +import base64 +import os +import pytest from pydantic import BaseModel - PROVIDER_TOOL_PROMPT_FORMAT = { - "remote::ollama": "python_list", + "remote::ollama": "json", "remote::together": "json", "remote::fireworks": "json", } @@ -28,34 +29,9 @@ def provider_tool_format(inference_provider_type): @pytest.fixture(scope="session") def inference_provider_type(llama_stack_client): providers = llama_stack_client.providers.list() - if "inference" not in providers: - pytest.fail("No inference providers available") - assert len(providers["inference"]) > 0 - return providers["inference"][0].provider_type - - -@pytest.fixture(scope="session") -def text_model_id(llama_stack_client): - available_models = [ - model.identifier - for model in llama_stack_client.models.list() - if model.identifier.startswith("meta-llama") and "405" not in model.identifier - ] - assert len(available_models) > 0 - return available_models[0] - - -@pytest.fixture(scope="session") -def vision_model_id(llama_stack_client): - available_models = [ - model.identifier - for model in llama_stack_client.models.list() - if "vision" in model.identifier.lower() - ] - if len(available_models) == 0: - pytest.skip("No vision models available") - - return available_models[0] + inference_providers = [p for p in providers if p.api == "inference"] + assert len(inference_providers) > 0, "No inference providers found" + return inference_providers[0].provider_type @pytest.fixture @@ -72,7 +48,17 @@ def get_weather_tool_definition(): } -def test_completion_non_streaming(llama_stack_client, text_model_id): +@pytest.fixture +def base64_image_url(): + image_path = os.path.join(os.path.dirname(__file__), "dog.png") + with open(image_path, "rb") as image_file: + # Convert the image to base64 + base64_string = base64.b64encode(image_file.read()).decode("utf-8") + base64_url = f"data:image/png;base64,{base64_string}" + return base64_url + + +def test_text_completion_non_streaming(llama_stack_client, text_model_id): response = llama_stack_client.inference.completion( content="Complete the sentence using one word: Roses are red, violets are ", stream=False, @@ -84,7 +70,7 @@ def test_completion_non_streaming(llama_stack_client, text_model_id): assert "blue" in response.content.lower().strip() -def test_completion_streaming(llama_stack_client, text_model_id): +def test_text_completion_streaming(llama_stack_client, text_model_id): response = llama_stack_client.inference.completion( content="Complete the sentence using one word: Roses are red, violets are ", stream=True, @@ -97,6 +83,7 @@ def test_completion_streaming(llama_stack_client, text_model_id): assert "blue" in "".join(streamed_content).lower().strip() +@pytest.mark.skip("Most inference providers don't support log probs yet") def test_completion_log_probs_non_streaming(llama_stack_client, text_model_id): response = llama_stack_client.inference.completion( content="Complete the sentence: Micheael Jordan is born in ", @@ -114,6 +101,7 @@ def test_completion_log_probs_non_streaming(llama_stack_client, text_model_id): assert all(len(logprob.logprobs_by_token) == 3 for logprob in response.logprobs) +@pytest.mark.skip("Most inference providers don't support log probs yet") def test_completion_log_probs_streaming(llama_stack_client, text_model_id): response = llama_stack_client.inference.completion( content="Complete the sentence: Micheael Jordan is born in ", @@ -137,7 +125,7 @@ def test_completion_log_probs_streaming(llama_stack_client, text_model_id): assert not chunk.logprobs, "Logprobs should be empty" -def test_completion_structured_output( +def test_text_completion_structured_output( llama_stack_client, text_model_id, inference_provider_type ): user_input = """ @@ -232,7 +220,6 @@ def test_text_chat_completion_with_tool_calling_and_non_streaming( # response to be a tool call assert response.completion_message.content == "" assert response.completion_message.role == "assistant" - assert response.completion_message.stop_reason == "end_of_turn" assert len(response.completion_message.tool_calls) == 1 assert response.completion_message.tool_calls[0].tool_name == "get_weather" @@ -245,19 +232,13 @@ def test_text_chat_completion_with_tool_calling_and_non_streaming( # The returned tool inovcation content will be a string so it's easy to comapare with expected value # e.g. "[get_weather, {'location': 'San Francisco, CA'}]" def extract_tool_invocation_content(response): - text_content: str = "" tool_invocation_content: str = "" for chunk in response: delta = chunk.event.delta - if delta.type == "text": - text_content += delta.text - elif delta.type == "tool_call": - if isinstance(delta.content, str): - tool_invocation_content += delta.content - else: - call = delta.content - tool_invocation_content += f"[{call.tool_name}, {call.arguments}]" - return text_content, tool_invocation_content + if delta.type == "tool_call" and delta.parse_status == "succeeded": + call = delta.tool_call + tool_invocation_content += f"[{call.tool_name}, {call.arguments}]" + return tool_invocation_content def test_text_chat_completion_with_tool_calling_and_streaming( @@ -274,8 +255,7 @@ def test_text_chat_completion_with_tool_calling_and_streaming( tool_prompt_format=provider_tool_format, stream=True, ) - text_content, tool_invocation_content = extract_tool_invocation_content(response) - + tool_invocation_content = extract_tool_invocation_content(response) assert tool_invocation_content == "[get_weather, {'location': 'San Francisco, CA'}]" @@ -319,9 +299,11 @@ def test_image_chat_completion_non_streaming(llama_stack_client, vision_model_id "content": [ { "type": "image", - "url": { - # TODO: Replace with Github based URI to resources/sample1.jpg - "uri": "https://www.healthypawspetinsurance.com/Images/V3/DogAndPuppyInsurance/Dog_CTA_Desktop_HeroImage.jpg" + "image": { + "url": { + # TODO: Replace with Github based URI to resources/sample1.jpg + "uri": "https://www.healthypawspetinsurance.com/Images/V3/DogAndPuppyInsurance/Dog_CTA_Desktop_HeroImage.jpg" + }, }, }, { @@ -346,9 +328,11 @@ def test_image_chat_completion_streaming(llama_stack_client, vision_model_id): "content": [ { "type": "image", - "url": { - # TODO: Replace with Github based URI to resources/sample1.jpg - "uri": "https://www.healthypawspetinsurance.com/Images/V3/DogAndPuppyInsurance/Dog_CTA_Desktop_HeroImage.jpg" + "image": { + "url": { + # TODO: Replace with Github based URI to resources/sample1.jpg + "uri": "https://www.healthypawspetinsurance.com/Images/V3/DogAndPuppyInsurance/Dog_CTA_Desktop_HeroImage.jpg" + }, }, }, { @@ -362,8 +346,37 @@ def test_image_chat_completion_streaming(llama_stack_client, vision_model_id): messages=[message], stream=True, ) - streamed_content = [ - str(chunk.event.delta.text.lower().strip()) for chunk in response - ] + streamed_content = "" + for chunk in response: + streamed_content += chunk.event.delta.text.lower() assert len(streamed_content) > 0 assert any(expected in streamed_content for expected in {"dog", "puppy", "pup"}) + + +def test_image_chat_completion_base64_url( + llama_stack_client, vision_model_id, base64_image_url +): + message = { + "role": "user", + "content": [ + { + "type": "image", + "image": { + "url": { + "uri": base64_image_url, + }, + }, + }, + { + "type": "text", + "text": "Describe what is in this image.", + }, + ], + } + response = llama_stack_client.inference.chat_completion( + model_id=vision_model_id, + messages=[message], + stream=False, + ) + message_content = response.completion_message.content.lower().strip() + assert len(message_content) > 0 diff --git a/tests/client-sdk/memory/test_memory.py b/tests/client-sdk/memory/test_memory.py deleted file mode 100644 index 998c30125..000000000 --- a/tests/client-sdk/memory/test_memory.py +++ /dev/null @@ -1,259 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the terms described in the LICENSE file in -# the root directory of this source tree. - -import random - -import pytest -from llama_stack.apis.memory import MemoryBankDocument - -from llama_stack_client.types.memory_insert_params import Document - - -@pytest.fixture(scope="function") -def empty_memory_bank_registry(llama_stack_client): - memory_banks = [ - memory_bank.identifier for memory_bank in llama_stack_client.memory_banks.list() - ] - for memory_bank_id in memory_banks: - llama_stack_client.memory_banks.unregister(memory_bank_id=memory_bank_id) - - -@pytest.fixture(scope="function") -def single_entry_memory_bank_registry(llama_stack_client, empty_memory_bank_registry): - memory_bank_id = f"test_bank_{random.randint(1000, 9999)}" - llama_stack_client.memory_banks.register( - memory_bank_id=memory_bank_id, - params={ - "memory_bank_type": "vector", - "embedding_model": "all-MiniLM-L6-v2", - "chunk_size_in_tokens": 512, - "overlap_size_in_tokens": 64, - }, - provider_id="faiss", - ) - memory_banks = [ - memory_bank.identifier for memory_bank in llama_stack_client.memory_banks.list() - ] - return memory_banks - - -@pytest.fixture(scope="session") -def sample_documents(): - return [ - MemoryBankDocument( - document_id="test-doc-1", - content="Python is a high-level programming language.", - metadata={"category": "programming", "difficulty": "beginner"}, - ), - MemoryBankDocument( - document_id="test-doc-2", - content="Machine learning is a subset of artificial intelligence.", - metadata={"category": "AI", "difficulty": "advanced"}, - ), - MemoryBankDocument( - document_id="test-doc-3", - content="Data structures are fundamental to computer science.", - metadata={"category": "computer science", "difficulty": "intermediate"}, - ), - MemoryBankDocument( - document_id="test-doc-4", - content="Neural networks are inspired by biological neural networks.", - metadata={"category": "AI", "difficulty": "advanced"}, - ), - ] - - -def assert_valid_response(response): - assert len(response.chunks) > 0 - assert len(response.scores) > 0 - assert len(response.chunks) == len(response.scores) - for chunk in response.chunks: - assert isinstance(chunk.content, str) - assert chunk.document_id is not None - - -def test_memory_bank_retrieve(llama_stack_client, empty_memory_bank_registry): - # Register a memory bank first - memory_bank_id = f"test_bank_{random.randint(1000, 9999)}" - llama_stack_client.memory_banks.register( - memory_bank_id=memory_bank_id, - params={ - "memory_bank_type": "vector", - "embedding_model": "all-MiniLM-L6-v2", - "chunk_size_in_tokens": 512, - "overlap_size_in_tokens": 64, - }, - provider_id="faiss", - ) - - # Retrieve the memory bank and validate its properties - response = llama_stack_client.memory_banks.retrieve(memory_bank_id=memory_bank_id) - assert response is not None - assert response.identifier == memory_bank_id - assert response.type == "memory_bank" - assert response.memory_bank_type == "vector" - assert response.embedding_model == "all-MiniLM-L6-v2" - assert response.chunk_size_in_tokens == 512 - assert response.overlap_size_in_tokens == 64 - assert response.provider_id == "faiss" - assert response.provider_resource_id == memory_bank_id - - -def test_memory_bank_list(llama_stack_client, empty_memory_bank_registry): - memory_banks_after_register = [ - memory_bank.identifier for memory_bank in llama_stack_client.memory_banks.list() - ] - assert len(memory_banks_after_register) == 0 - - -def test_memory_bank_register(llama_stack_client, empty_memory_bank_registry): - memory_provider_id = "faiss" - memory_bank_id = f"test_bank_{random.randint(1000, 9999)}" - llama_stack_client.memory_banks.register( - memory_bank_id=memory_bank_id, - params={ - "memory_bank_type": "vector", - "embedding_model": "all-MiniLM-L6-v2", - "chunk_size_in_tokens": 512, - "overlap_size_in_tokens": 64, - }, - provider_id=memory_provider_id, - ) - - memory_banks_after_register = [ - memory_bank.identifier for memory_bank in llama_stack_client.memory_banks.list() - ] - assert memory_banks_after_register == [memory_bank_id] - - -def test_memory_bank_unregister(llama_stack_client, single_entry_memory_bank_registry): - memory_banks = [ - memory_bank.identifier for memory_bank in llama_stack_client.memory_banks.list() - ] - assert len(memory_banks) == 1 - - memory_bank_id = memory_banks[0] - llama_stack_client.memory_banks.unregister(memory_bank_id=memory_bank_id) - - memory_banks = [ - memory_bank.identifier for memory_bank in llama_stack_client.memory_banks.list() - ] - assert len(memory_banks) == 0 - - -def test_memory_bank_insert_inline_and_query( - llama_stack_client, single_entry_memory_bank_registry, sample_documents -): - memory_bank_id = single_entry_memory_bank_registry[0] - llama_stack_client.memory.insert( - bank_id=memory_bank_id, - documents=sample_documents, - ) - - # Query with a direct match - query1 = "programming language" - response1 = llama_stack_client.memory.query( - bank_id=memory_bank_id, - query=query1, - ) - assert_valid_response(response1) - assert any("Python" in chunk.content for chunk in response1.chunks) - - # Query with semantic similarity - query2 = "AI and brain-inspired computing" - response2 = llama_stack_client.memory.query( - bank_id=memory_bank_id, - query=query2, - ) - assert_valid_response(response2) - assert any("neural networks" in chunk.content.lower() for chunk in response2.chunks) - - # Query with limit on number of results (max_chunks=2) - query3 = "computer" - response3 = llama_stack_client.memory.query( - bank_id=memory_bank_id, - query=query3, - params={"max_chunks": 2}, - ) - assert_valid_response(response3) - assert len(response3.chunks) <= 2 - - # Query with threshold on similarity score - query4 = "computer" - response4 = llama_stack_client.memory.query( - bank_id=memory_bank_id, - query=query4, - params={"score_threshold": 0.01}, - ) - assert_valid_response(response4) - assert all(score >= 0.01 for score in response4.scores) - - -def test_memory_bank_insert_from_url_and_query( - llama_stack_client, empty_memory_bank_registry -): - providers = llama_stack_client.providers.list() - assert "memory" in providers - assert len(providers["memory"]) > 0 - - memory_provider_id = providers["memory"][0].provider_id - memory_bank_id = "test_bank" - - llama_stack_client.memory_banks.register( - memory_bank_id=memory_bank_id, - params={ - "memory_bank_type": "vector", - "embedding_model": "all-MiniLM-L6-v2", - "chunk_size_in_tokens": 512, - "overlap_size_in_tokens": 64, - }, - provider_id=memory_provider_id, - ) - - # list to check memory bank is successfully registered - available_memory_banks = [ - memory_bank.identifier for memory_bank in llama_stack_client.memory_banks.list() - ] - assert memory_bank_id in available_memory_banks - - # URLs of documents to insert - # TODO: Move to test/memory/resources then update the url to - # https://raw.githubusercontent.com/meta-llama/llama-stack/main/tests/memory/resources/{url} - urls = [ - "memory_optimizations.rst", - "chat.rst", - "llama3.rst", - ] - documents = [ - Document( - document_id=f"num-{i}", - content=f"https://raw.githubusercontent.com/pytorch/torchtune/main/docs/source/tutorials/{url}", - mime_type="text/plain", - metadata={}, - ) - for i, url in enumerate(urls) - ] - - llama_stack_client.memory.insert( - bank_id=memory_bank_id, - documents=documents, - ) - - # Query for the name of method - response1 = llama_stack_client.memory.query( - bank_id=memory_bank_id, - query="What's the name of the fine-tunning method used?", - ) - assert_valid_response(response1) - assert any("lora" in chunk.content.lower() for chunk in response1.chunks) - - # Query for the name of model - response2 = llama_stack_client.memory.query( - bank_id=memory_bank_id, - query="Which Llama model is mentioned?", - ) - assert_valid_response(response1) - assert any("llama2" in chunk.content.lower() for chunk in response2.chunks) diff --git a/tests/client-sdk/metadata.py b/tests/client-sdk/metadata.py new file mode 100644 index 000000000..badd7edff --- /dev/null +++ b/tests/client-sdk/metadata.py @@ -0,0 +1,50 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +from llama_stack.providers.datatypes import Api + +INFERENCE_API_CAPA_TEST_MAP = { + "chat_completion": { + "streaming": [ + "test_text_chat_completion_streaming", + "test_image_chat_completion_streaming", + ], + "non_streaming": [ + "test_image_chat_completion_non_streaming", + "test_text_chat_completion_non_streaming", + ], + "tool_calling": [ + "test_text_chat_completion_with_tool_calling_and_streaming", + "test_text_chat_completion_with_tool_calling_and_non_streaming", + ], + }, + "completion": { + "streaming": ["test_text_completion_streaming"], + "non_streaming": ["test_text_completion_non_streaming"], + "structured_output": ["test_text_completion_structured_output"], + }, +} + +VECTORIO_API_TEST_MAP = { + "retrieve": { + "": ["test_vector_db_retrieve"], + } +} + +AGENTS_API_TEST_MAP = { + "create_agent_turn": { + "rag": ["test_rag_agent"], + "custom_tool": ["test_custom_tool"], + "code_execution": ["test_code_interpreter_for_attachments"], + } +} + + +API_MAPS = { + Api.inference: INFERENCE_API_CAPA_TEST_MAP, + Api.vector_io: VECTORIO_API_TEST_MAP, + Api.agents: AGENTS_API_TEST_MAP, +} diff --git a/tests/client-sdk/report.py b/tests/client-sdk/report.py new file mode 100644 index 000000000..cf7a84d7f --- /dev/null +++ b/tests/client-sdk/report.py @@ -0,0 +1,265 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + + +import importlib +import os +from collections import defaultdict +from pathlib import Path +from urllib.parse import urlparse + +import pytest +from llama_models.datatypes import CoreModelId +from llama_models.sku_list import ( + all_registered_models, + llama3_1_instruct_models, + llama3_2_instruct_models, + llama3_3_instruct_models, + llama3_instruct_models, + safety_models, +) + +from llama_stack.distribution.library_client import LlamaStackAsLibraryClient +from llama_stack.providers.datatypes import Api +from llama_stack.providers.tests.env import get_env_or_fail + +from llama_stack_client import LlamaStackClient +from metadata import API_MAPS + +from pytest import CollectReport +from termcolor import cprint + + +def featured_models_repo_names(): + models = [ + *llama3_instruct_models(), + *llama3_1_instruct_models(), + *llama3_2_instruct_models(), + *llama3_3_instruct_models(), + *safety_models(), + ] + return [model.huggingface_repo for model in models if not model.variant] + + +SUPPORTED_MODELS = { + "ollama": set( + [ + CoreModelId.llama3_1_8b_instruct.value, + CoreModelId.llama3_1_8b_instruct.value, + CoreModelId.llama3_1_70b_instruct.value, + CoreModelId.llama3_1_70b_instruct.value, + CoreModelId.llama3_1_405b_instruct.value, + CoreModelId.llama3_1_405b_instruct.value, + CoreModelId.llama3_2_1b_instruct.value, + CoreModelId.llama3_2_1b_instruct.value, + CoreModelId.llama3_2_3b_instruct.value, + CoreModelId.llama3_2_3b_instruct.value, + CoreModelId.llama3_2_11b_vision_instruct.value, + CoreModelId.llama3_2_11b_vision_instruct.value, + CoreModelId.llama3_2_90b_vision_instruct.value, + CoreModelId.llama3_2_90b_vision_instruct.value, + CoreModelId.llama3_3_70b_instruct.value, + CoreModelId.llama_guard_3_8b.value, + CoreModelId.llama_guard_3_1b.value, + ] + ), + "tgi": set( + [ + model.core_model_id.value + for model in all_registered_models() + if model.huggingface_repo + ] + ), + "vllm": set( + [ + model.core_model_id.value + for model in all_registered_models() + if model.huggingface_repo + ] + ), +} + + +class Report: + + def __init__(self): + if os.environ.get("LLAMA_STACK_CONFIG"): + config_path_or_template_name = get_env_or_fail("LLAMA_STACK_CONFIG") + if config_path_or_template_name.endswith(".yaml"): + config_path = Path(config_path_or_template_name) + else: + config_path = Path( + importlib.resources.files("llama_stack") + / f"templates/{config_path_or_template_name}/run.yaml" + ) + if not config_path.exists(): + raise ValueError(f"Config file {config_path} does not exist") + self.output_path = Path(config_path.parent / "report.md") + self.client = LlamaStackAsLibraryClient( + config_path_or_template_name, + provider_data=None, + skip_logger_removal=True, + ) + self.client.initialize() + self.image_name = self.client.async_client.config.image_name + elif os.environ.get("LLAMA_STACK_BASE_URL"): + url = get_env_or_fail("LLAMA_STACK_BASE_URL") + hostname = urlparse(url).netloc + domain = hostname.split(".")[-2] + self.image_name = domain + + self.client = LlamaStackClient( + base_url=url, + provider_data=None, + ) + # We assume that the domain maps to a template + # i.e. https://llamastack-preview.fireworks.ai --> "fireworks" template + # and add report in that directory + output_dir = Path( + importlib.resources.files("llama_stack") / f"templates/{domain}/" + ) + if not output_dir.exists(): + raise ValueError(f"Output dir {output_dir} does not exist") + self.output_path = Path(output_dir / "remote-hosted-report.md") + else: + raise ValueError("LLAMA_STACK_CONFIG or LLAMA_STACK_BASE_URL must be set") + + self.report_data = defaultdict(dict) + # test function -> test nodeid + self.test_data = dict() + self.test_name_to_nodeid = defaultdict(list) + self.vision_model_id = None + self.text_model_id = None + + @pytest.hookimpl(tryfirst=True) + def pytest_runtest_logreport(self, report): + # This hook is called in several phases, including setup, call and teardown + # The test is considered failed / error if any of the outcomes is not "Passed" + outcome = self._process_outcome(report) + if report.nodeid not in self.test_data: + self.test_data[report.nodeid] = outcome + elif self.test_data[report.nodeid] != outcome and outcome != "Passed": + self.test_data[report.nodeid] = outcome + + def pytest_sessionfinish(self, session): + report = [] + report.append(f"# Report for {self.image_name} distribution") + report.append("\n## Supported Models") + + header = f"| Model Descriptor | {self.image_name} |" + dividor = "|:---|:---|" + + report.append(header) + report.append(dividor) + + rows = [] + if self.image_name in SUPPORTED_MODELS: + for model in all_registered_models(): + if ( + "Instruct" not in model.core_model_id.value + and "Guard" not in model.core_model_id.value + ) or (model.variant): + continue + row = f"| {model.core_model_id.value} |" + if model.core_model_id.value in SUPPORTED_MODELS[self.image_name]: + row += " ✅ |" + else: + row += " ❌ |" + rows.append(row) + else: + supported_models = {m.identifier for m in self.client.models.list()} + for model in featured_models_repo_names(): + row = f"| {model} |" + if model in supported_models: + row += " ✅ |" + else: + row += " ❌ |" + rows.append(row) + report.extend(rows) + + report.append("\n## Inference") + test_table = [ + "| Model | API | Capability | Test | Status |", + "|:----- |:-----|:-----|:-----|:-----|", + ] + for api, capa_map in API_MAPS[Api.inference].items(): + for capa, tests in capa_map.items(): + for test_name in tests: + model_id = ( + self.text_model_id + if "text" in test_name + else self.vision_model_id + ) + test_nodeids = self.test_name_to_nodeid[test_name] + assert len(test_nodeids) > 0 + + # There might be more than one parametrizations for the same test function. We take + # the result of the first one for now. Ideally we should mark the test as failed if + # any of the parametrizations failed. + test_table.append( + f"| {model_id} | /{api} | {capa} | {test_name} | {self._print_result_icon(self.test_data[test_nodeids[0]])} |" + ) + + report.extend(test_table) + + name_map = {Api.vector_io: "Vector IO", Api.agents: "Agents"} + for api_group in [Api.vector_io, Api.agents]: + api_capitalized = name_map[api_group] + report.append(f"\n## {api_capitalized}") + test_table = [ + "| API | Capability | Test | Status |", + "|:-----|:-----|:-----|:-----|", + ] + for api, capa_map in API_MAPS[api_group].items(): + for capa, tests in capa_map.items(): + for test_name in tests: + test_nodeids = self.test_name_to_nodeid[test_name] + assert len(test_nodeids) > 0 + test_table.append( + f"| /{api} | {capa} | {test_name} | {self._print_result_icon(self.test_data[test_nodeids[0]])} |" + ) + report.extend(test_table) + + output_file = self.output_path + text = "\n".join(report) + "\n" + output_file.write_text(text) + cprint(f"\nReport generated: {output_file.absolute()}", "green") + + def pytest_runtest_makereport(self, item, call): + func_name = getattr(item, "originalname", item.name) + if "text_model_id" in item.funcargs: + text_model = item.funcargs["text_model_id"].split("/")[1] + self.text_model_id = self.text_model_id or text_model + elif "vision_model_id" in item.funcargs: + vision_model = item.funcargs["vision_model_id"].split("/")[1] + self.vision_model_id = self.vision_model_id or vision_model + + self.test_name_to_nodeid[func_name].append(item.nodeid) + + def _print_result_icon(self, result): + if result == "Passed": + return "✅" + elif result == "Failed" or result == "Error": + return "❌" + else: + # result == "Skipped": + return "⏭️" + + def _process_outcome(self, report: CollectReport): + if self._is_error(report): + return "Error" + if hasattr(report, "wasxfail"): + if report.outcome in ["passed", "failed"]: + return "XPassed" + if report.outcome == "skipped": + return "XFailed" + return report.outcome.capitalize() + + def _is_error(self, report: CollectReport): + return ( + report.when in ["setup", "teardown", "collect"] + and report.outcome == "failed" + ) diff --git a/tests/client-sdk/safety/conftest.py b/tests/client-sdk/safety/conftest.py new file mode 100644 index 000000000..c4570801c --- /dev/null +++ b/tests/client-sdk/safety/conftest.py @@ -0,0 +1,22 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + + +def pytest_addoption(parser): + parser.addoption( + "--safety-shield", + action="store", + default="meta-llama/Llama-Guard-3-1B", + help="Specify the safety shield model to use for testing", + ) + + +def pytest_generate_tests(metafunc): + if "llama_guard_text_shield_id" in metafunc.fixturenames: + metafunc.parametrize( + "llama_guard_text_shield_id", + [metafunc.config.getoption("--safety-shield")], + ) diff --git a/tests/client-sdk/safety/test_safety.py b/tests/client-sdk/safety/test_safety.py index 8eadffcfc..7456fb88f 100644 --- a/tests/client-sdk/safety/test_safety.py +++ b/tests/client-sdk/safety/test_safety.py @@ -11,7 +11,6 @@ import pytest from llama_stack.apis.safety import ViolationLevel - VISION_SHIELD_ENABLED_PROVIDERS = {"together"} CODE_SCANNER_ENABLED_PROVIDERS = {"ollama", "together", "fireworks"} @@ -33,16 +32,6 @@ def available_shields(llama_stack_client): return [shield.identifier for shield in llama_stack_client.shields.list()] -@pytest.fixture(scope="session") -def llama_guard_text_shield_id(available_shields): - if "meta-llama/Llama-Guard-3-1B" in available_shields: - return "meta-llama/Llama-Guard-3-1B" - elif "meta-llama/Llama-Guard-3-8B" in available_shields: - return "meta-llama/Llama-Guard-3-8B" - else: - pytest.skip("Llama-Guard shield is not available. Skipping.") - - @pytest.fixture(scope="session") def code_scanner_shield_id(available_shields): if "CodeScanner" in available_shields: @@ -54,7 +43,11 @@ def code_scanner_shield_id(available_shields): @pytest.fixture(scope="session") def model_providers(llama_stack_client): return set( - [x.provider_id for x in llama_stack_client.providers.list()["inference"]] + [ + x.provider_id + for x in llama_stack_client.providers.list() + if x.api == "inference" + ] ) @@ -138,7 +131,7 @@ def test_safety_with_image(llama_stack_client, model_providers): }, { "type": "image", - "url": {"uri": data_url_from_image(file_path)}, + "image": {"url": {"uri": data_url_from_image(file_path)}}, }, ], } diff --git a/tests/client-sdk/tool_runtime/test_rag_tool.py b/tests/client-sdk/tool_runtime/test_rag_tool.py new file mode 100644 index 000000000..6e158a1e3 --- /dev/null +++ b/tests/client-sdk/tool_runtime/test_rag_tool.py @@ -0,0 +1,180 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +import random + +import pytest + +from llama_stack_client.types import Document + + +@pytest.fixture(scope="function") +def empty_vector_db_registry(llama_stack_client): + vector_dbs = [ + vector_db.identifier for vector_db in llama_stack_client.vector_dbs.list() + ] + for vector_db_id in vector_dbs: + llama_stack_client.vector_dbs.unregister(vector_db_id=vector_db_id) + + +@pytest.fixture(scope="function") +def single_entry_vector_db_registry(llama_stack_client, empty_vector_db_registry): + vector_db_id = f"test_vector_db_{random.randint(1000, 9999)}" + llama_stack_client.vector_dbs.register( + vector_db_id=vector_db_id, + embedding_model="all-MiniLM-L6-v2", + embedding_dimension=384, + provider_id="faiss", + ) + vector_dbs = [ + vector_db.identifier for vector_db in llama_stack_client.vector_dbs.list() + ] + return vector_dbs + + +@pytest.fixture(scope="session") +def sample_documents(): + return [ + Document( + document_id="test-doc-1", + content="Python is a high-level programming language.", + metadata={"category": "programming", "difficulty": "beginner"}, + ), + Document( + document_id="test-doc-2", + content="Machine learning is a subset of artificial intelligence.", + metadata={"category": "AI", "difficulty": "advanced"}, + ), + Document( + document_id="test-doc-3", + content="Data structures are fundamental to computer science.", + metadata={"category": "computer science", "difficulty": "intermediate"}, + ), + Document( + document_id="test-doc-4", + content="Neural networks are inspired by biological neural networks.", + metadata={"category": "AI", "difficulty": "advanced"}, + ), + ] + + +def assert_valid_response(response): + assert len(response.chunks) > 0 + assert len(response.scores) > 0 + assert len(response.chunks) == len(response.scores) + for chunk in response.chunks: + assert isinstance(chunk.content, str) + + +def test_vector_db_insert_inline_and_query( + llama_stack_client, single_entry_vector_db_registry, sample_documents +): + vector_db_id = single_entry_vector_db_registry[0] + llama_stack_client.tool_runtime.rag_tool.insert( + documents=sample_documents, + chunk_size_in_tokens=512, + vector_db_id=vector_db_id, + ) + + # Query with a direct match + query1 = "programming language" + response1 = llama_stack_client.vector_io.query( + vector_db_id=vector_db_id, + query=query1, + ) + assert_valid_response(response1) + assert any("Python" in chunk.content for chunk in response1.chunks) + + # Query with semantic similarity + query2 = "AI and brain-inspired computing" + response2 = llama_stack_client.vector_io.query( + vector_db_id=vector_db_id, + query=query2, + ) + assert_valid_response(response2) + assert any("neural networks" in chunk.content.lower() for chunk in response2.chunks) + + # Query with limit on number of results (max_chunks=2) + query3 = "computer" + response3 = llama_stack_client.vector_io.query( + vector_db_id=vector_db_id, + query=query3, + params={"max_chunks": 2}, + ) + assert_valid_response(response3) + assert len(response3.chunks) <= 2 + + # Query with threshold on similarity score + query4 = "computer" + response4 = llama_stack_client.vector_io.query( + vector_db_id=vector_db_id, + query=query4, + params={"score_threshold": 0.01}, + ) + assert_valid_response(response4) + assert all(score >= 0.01 for score in response4.scores) + + +def test_vector_db_insert_from_url_and_query( + llama_stack_client, empty_vector_db_registry +): + providers = [p for p in llama_stack_client.providers.list() if p.api == "vector_io"] + assert len(providers) > 0 + + vector_db_id = "test_vector_db" + + llama_stack_client.vector_dbs.register( + vector_db_id=vector_db_id, + embedding_model="all-MiniLM-L6-v2", + embedding_dimension=384, + provider_id="faiss", + ) + + # list to check memory bank is successfully registered + available_vector_dbs = [ + vector_db.identifier for vector_db in llama_stack_client.vector_dbs.list() + ] + assert vector_db_id in available_vector_dbs + + # URLs of documents to insert + # TODO: Move to test/memory/resources then update the url to + # https://raw.githubusercontent.com/meta-llama/llama-stack/main/tests/memory/resources/{url} + urls = [ + "memory_optimizations.rst", + "chat.rst", + "llama3.rst", + ] + documents = [ + Document( + document_id=f"num-{i}", + content=f"https://raw.githubusercontent.com/pytorch/torchtune/main/docs/source/tutorials/{url}", + mime_type="text/plain", + metadata={}, + ) + for i, url in enumerate(urls) + ] + + llama_stack_client.tool_runtime.rag_tool.insert( + documents=documents, + vector_db_id=vector_db_id, + chunk_size_in_tokens=512, + ) + + # Query for the name of method + response1 = llama_stack_client.vector_io.query( + vector_db_id=vector_db_id, + query="What's the name of the fine-tunning method used?", + ) + assert_valid_response(response1) + assert any("lora" in chunk.content.lower() for chunk in response1.chunks) + + # Query for the name of model + response2 = llama_stack_client.vector_io.query( + vector_db_id=vector_db_id, + query="Which Llama model is mentioned?", + ) + assert_valid_response(response2) + assert any("llama2" in chunk.content.lower() for chunk in response2.chunks) diff --git a/llama_stack/apis/memory/__init__.py b/tests/client-sdk/vector_io/__init__.py similarity index 82% rename from llama_stack/apis/memory/__init__.py rename to tests/client-sdk/vector_io/__init__.py index 260862228..756f351d8 100644 --- a/llama_stack/apis/memory/__init__.py +++ b/tests/client-sdk/vector_io/__init__.py @@ -3,5 +3,3 @@ # # This source code is licensed under the terms described in the LICENSE file in # the root directory of this source tree. - -from .memory import * # noqa: F401 F403 diff --git a/tests/client-sdk/vector_io/conftest.py b/tests/client-sdk/vector_io/conftest.py new file mode 100644 index 000000000..64cac27d2 --- /dev/null +++ b/tests/client-sdk/vector_io/conftest.py @@ -0,0 +1,22 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + + +def pytest_addoption(parser): + parser.addoption( + "--embedding-model", + action="store", + default="all-MiniLM-L6-v2", + help="Specify the embedding model to use for testing", + ) + + +def pytest_generate_tests(metafunc): + if "embedding_model" in metafunc.fixturenames: + metafunc.parametrize( + "embedding_model", + [metafunc.config.getoption("--embedding-model")], + ) diff --git a/tests/client-sdk/vector_io/test_vector_io.py b/tests/client-sdk/vector_io/test_vector_io.py new file mode 100644 index 000000000..2a110b73a --- /dev/null +++ b/tests/client-sdk/vector_io/test_vector_io.py @@ -0,0 +1,93 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +import random + +import pytest + + +@pytest.fixture(scope="function") +def empty_vector_db_registry(llama_stack_client): + vector_dbs = [ + vector_db.identifier for vector_db in llama_stack_client.vector_dbs.list() + ] + for vector_db_id in vector_dbs: + llama_stack_client.vector_dbs.unregister(vector_db_id=vector_db_id) + + +@pytest.fixture(scope="function") +def single_entry_vector_db_registry(llama_stack_client, empty_vector_db_registry): + vector_db_id = f"test_vector_db_{random.randint(1000, 9999)}" + llama_stack_client.vector_dbs.register( + vector_db_id=vector_db_id, + embedding_model="all-MiniLM-L6-v2", + embedding_dimension=384, + provider_id="faiss", + ) + vector_dbs = [ + vector_db.identifier for vector_db in llama_stack_client.vector_dbs.list() + ] + return vector_dbs + + +def test_vector_db_retrieve( + llama_stack_client, embedding_model, empty_vector_db_registry +): + # Register a memory bank first + vector_db_id = f"test_vector_db_{random.randint(1000, 9999)}" + llama_stack_client.vector_dbs.register( + vector_db_id=vector_db_id, + embedding_model=embedding_model, + embedding_dimension=384, + provider_id="faiss", + ) + + # Retrieve the memory bank and validate its properties + response = llama_stack_client.vector_dbs.retrieve(vector_db_id=vector_db_id) + assert response is not None + assert response.identifier == vector_db_id + assert response.embedding_model == embedding_model + assert response.provider_id == "faiss" + assert response.provider_resource_id == vector_db_id + + +def test_vector_db_list(llama_stack_client, empty_vector_db_registry): + vector_dbs_after_register = [ + vector_db.identifier for vector_db in llama_stack_client.vector_dbs.list() + ] + assert len(vector_dbs_after_register) == 0 + + +def test_vector_db_register( + llama_stack_client, embedding_model, empty_vector_db_registry +): + vector_db_id = f"test_vector_db_{random.randint(1000, 9999)}" + llama_stack_client.vector_dbs.register( + vector_db_id=vector_db_id, + embedding_model=embedding_model, + embedding_dimension=384, + provider_id="faiss", + ) + + vector_dbs_after_register = [ + vector_db.identifier for vector_db in llama_stack_client.vector_dbs.list() + ] + assert vector_dbs_after_register == [vector_db_id] + + +def test_vector_db_unregister(llama_stack_client, single_entry_vector_db_registry): + vector_dbs = [ + vector_db.identifier for vector_db in llama_stack_client.vector_dbs.list() + ] + assert len(vector_dbs) == 1 + + vector_db_id = vector_dbs[0] + llama_stack_client.vector_dbs.unregister(vector_db_id=vector_db_id) + + vector_dbs = [ + vector_db.identifier for vector_db in llama_stack_client.vector_dbs.list() + ] + assert len(vector_dbs) == 0