Merge branch 'meta-llama:main' into main

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,17 +1,14 @@
 # What does this PR do?
 
-Closes # (issue)
+In short, provide a summary of what this PR does and why. Usually, the relevant context should be present in a linked issue.
+
+- [ ] Addresses issue (#issue)
 
 ## Feature/Issue validation/testing/test plan
 
-Please describe the tests that you ran to verify your changes and relevant result summary. Provide instructions so it can be reproduced.
+Please describe:
-Please also list any relevant details for your test configuration or test plan.
+ - tests you ran to verify your changes with result summaries.
-
+ - provide instructions so it can be reproduced.
-- [ ] Test A
-Logs for Test A
-
-- [ ] Test B
-Logs for Test B
 
 
 ## Sources
@@ -20,12 +17,10 @@ Please link relevant resources if necessary.
 
 
 ## Before submitting
-- [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case).
-- [ ] Did you read the [contributor guideline](https://github.com/meta-llama/llama-stack/blob/main/CONTRIBUTING.md),
-      Pull Request section?
-- [ ] Was this discussed/approved via a Github issue? Please add a link
-      to it if that's the case.
-- [ ] Did you make sure to update the documentation with your changes?
-- [ ] Did you write any new necessary tests?
 
-Thanks for contributing 🎉!
+- [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case).
+- [ ] Ran pre-commit to handle lint / formatting issues.
+- [ ] Read the [contributor guideline](https://github.com/meta-llama/llama-stack/blob/main/CONTRIBUTING.md),
+      Pull Request section?
+- [ ] Updated relevant documentation.
+- [ ] Wrote necessary unit or integration tests.

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "llama_stack/providers/impls/ios/inference/executorch"]
-	path = llama_stack/providers/impls/ios/inference/executorch
+	path = llama_stack/providers/inline/ios/inference/executorch
 	url = https://github.com/pytorch/executorch

distributions/bedrock/compose.yaml

@@ -0,0 +1,15 @@
+services:
+  llamastack:
+    image: distribution-bedrock
+    volumes:
+      - ~/.llama:/root/.llama
+      - ./run.yaml:/root/llamastack-run-bedrock.yaml
+    ports:
+      - "5000:5000"
+    entrypoint: bash -c "python -m llama_stack.distribution.server.server --yaml_config /root/llamastack-run-bedrock.yaml"
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 3s
+        max_attempts: 5
+        window: 60s

distributions/bedrock/run.yaml

@@ -0,0 +1,46 @@
+version: '2'
+built_at: '2024-11-01T17:40:45.325529'
+image_name: local
+name: bedrock
+docker_image: null
+conda_env: local
+apis:
+- shields
+- agents
+- models
+- memory
+- memory_banks
+- inference
+- safety
+providers:
+  inference:
+    - provider_id: bedrock0
+      provider_type: remote::bedrock
+      config:
+        aws_access_key_id: <AWS_ACCESS_KEY_ID>
+        aws_secret_access_key: <AWS_SECRET_ACCESS_KEY>
+        aws_session_token: <AWS_SESSION_TOKEN>
+        region_name: <AWS_REGION>
+  memory:
+    - provider_id: meta0
+      provider_type: meta-reference
+      config: {}
+  safety:
+    - provider_id: bedrock0
+      provider_type: remote::bedrock
+      config:
+        aws_access_key_id: <AWS_ACCESS_KEY_ID>
+        aws_secret_access_key: <AWS_SECRET_ACCESS_KEY>
+        aws_session_token: <AWS_SESSION_TOKEN>
+        region_name: <AWS_REGION>
+  agents:
+    - provider_id: meta0
+      provider_type: meta-reference
+      config:
+        persistence_store:
+          type: sqlite
+          db_path: ~/.llama/runtime/kvstore.db
+  telemetry:
+    - provider_id: meta0
+      provider_type: meta-reference
+      config: {}

docs/getting_started.ipynb

@@ -61,49 +61,7 @@
     "```\n",
     "For GPU inference, you need to set these environment variables for specifying local directory containing your model checkpoints, and enable GPU inference to start running docker container.\n",
     "$ export LLAMA_CHECKPOINT_DIR=~/.llama\n",
-    "$ llama stack configure llamastack-meta-reference-gpu\n",
     "```\n",
-    "Follow the prompts as part of configure.\n",
-    "Here is a sample output \n",
-    "```\n",
-    "$ llama stack configure llamastack-meta-reference-gpu\n",
-    "\n",
-    "Could not find ~/.conda/envs/llamastack-llamastack-meta-reference-gpu/llamastack-meta-reference-gpu-build.yaml. Trying docker image name instead...\n",
-    "+ podman run --network host -it -v ~/.llama/builds/docker:/app/builds llamastack-meta-reference-gpu llama stack configure ./llamastack-build.yaml --output-dir /app/builds\n",
-    "\n",
-    "Configuring API `inference`...\n",
-    "=== Configuring provider `meta-reference` for API inference...\n",
-    "Enter value for model (default: Llama3.1-8B-Instruct) (required): Llama3.2-11B-Vision-Instruct\n",
-    "Do you want to configure quantization? (y/n): n\n",
-    "Enter value for torch_seed (optional): \n",
-    "Enter value for max_seq_len (default: 4096) (required): \n",
-    "Enter value for max_batch_size (default: 1) (required): \n",
-    "\n",
-    "Configuring API `safety`...\n",
-    "=== Configuring provider `meta-reference` for API safety...\n",
-    "Do you want to configure llama_guard_shield? (y/n): n\n",
-    "Do you want to configure prompt_guard_shield? (y/n): n\n",
-    "\n",
-    "Configuring API `agents`...\n",
-    "=== Configuring provider `meta-reference` for API agents...\n",
-    "Enter `type` for persistence_store (options: redis, sqlite, postgres) (default: sqlite): \n",
-    "\n",
-    "Configuring SqliteKVStoreConfig:\n",
-    "Enter value for namespace (optional): \n",
-    "Enter value for db_path (default: /root/.llama/runtime/kvstore.db) (required): \n",
-    "\n",
-    "Configuring API `memory`...\n",
-    "=== Configuring provider `meta-reference` for API memory...\n",
-    "> Please enter the supported memory bank type your provider has for memory: vector\n",
-    "\n",
-    "Configuring API `telemetry`...\n",
-    "=== Configuring provider `meta-reference` for API telemetry...\n",
-    "\n",
-    "> YAML configuration has been written to /app/builds/local-gpu-run.yaml.\n",
-    "You can now run `llama stack run local-gpu --port PORT`\n",
-    "YAML configuration has been written to /home/hjshah/.llama/builds/docker/local-gpu-run.yaml. You can now run `llama stack run /home/hjshah/.llama/builds/docker/local-gpu-run.yaml`\n",
-    "```\n",
-    "NOTE: For this example, we use all local meta-reference implementations and have not setup safety. \n",
     "\n",
     "5.  Run the Stack Server\n",
     "```\n",

docs/source/api_providers/new_api_provider.md

@@ -6,8 +6,8 @@ This guide contains references to walk you through adding a new API provider.
 1. First, decide which API your provider falls into (e.g. Inference, Safety, Agents, Memory).
 2. Decide whether your provider is a remote provider, or inline implmentation. A remote provider is a provider that makes a remote request to an service. An inline provider is a provider where implementation is executed locally. Checkout the examples, and follow the structure to add your own API provider. Please find the following code pointers:
 
-    - [Inference Remote Adapter](https://github.com/meta-llama/llama-stack/tree/docs/llama_stack/providers/adapters/inference)
+    - [Inference Remote Adapter](https://github.com/meta-llama/llama-stack/tree/docs/llama_stack/providers/remote/inference)
-    - [Inference Inline Provider](https://github.com/meta-llama/llama-stack/tree/docs/llama_stack/providers/impls/meta_reference/inference)
+    - [Inference Inline Provider](https://github.com/meta-llama/llama-stack/tree/docs/llama_stack/providers/inline/meta_reference/inference)
 
 3. [Build a Llama Stack distribution](https://llama-stack.readthedocs.io/en/latest/distribution_dev/building_distro.html) with your API provider.
 4. Test your code!

docs/source/distribution_dev/building_distro.md

@@ -1,53 +1,56 @@
 # Developer Guide: Assemble a Llama Stack Distribution
 
-> NOTE: This doc may be out-of-date.
 
-This guide will walk you through the steps to get started with building a Llama Stack distributiom from scratch with your choice of API providers. Please see the [Getting Started Guide](./getting_started.md) if you just want the basic steps to start a Llama Stack distribution.
+This guide will walk you through the steps to get started with building a Llama Stack distributiom from scratch with your choice of API providers. Please see the [Getting Started Guide](https://llama-stack.readthedocs.io/en/latest/getting_started/index.html) if you just want the basic steps to start a Llama Stack distribution.
 
 ## Step 1. Build
-In the following steps, imagine we'll be working with a `Meta-Llama3.1-8B-Instruct` model. We will name our build `8b-instruct` to help us remember the config. We will start build our distribution (in the form of a Conda environment, or Docker image). In this step, we will specify:
+
-- `name`: the name for our distribution (e.g. `8b-instruct`)
+### Llama Stack Build Options
+
+```
+llama stack build -h
+```
+We will start build our distribution (in the form of a Conda environment, or Docker image). In this step, we will specify:
+- `name`: the name for our distribution (e.g. `my-stack`)
 - `image_type`: our build image type (`conda | docker`)
 - `distribution_spec`: our distribution specs for specifying API providers
   - `description`: a short description of the configurations for the distribution
   - `providers`: specifies the underlying implementation for serving each API endpoint
   - `image_type`: `conda` | `docker` to specify whether to build the distribution in the form of Docker image or Conda environment.
 
+After this step is complete, a file named `<name>-build.yaml` and template file `<name>-run.yaml` will be generated and saved at the output file path specified at the end of the command.
 
-At the end of build command, we will generate `<name>-build.yaml` file storing the build configurations.
+::::{tab-set}
+:::{tab-item} Building from Scratch
 
-After this step is complete, a file named `<name>-build.yaml` will be generated and saved at the output file path specified at the end of the command.
-
-#### Building from scratch
 - For a new user, we could start off with running `llama stack build` which will allow you to a interactively enter wizard where you will be prompted to enter build configurations.
 ```
 llama stack build
+
+> Enter a name for your Llama Stack (e.g. my-local-stack): my-stack
+> Enter the image type you want your Llama Stack to be built as (docker or conda): conda
+
+Llama Stack is composed of several APIs working together. Let's select
+the provider types (implementations) you want to use for these APIs.
+
+Tip: use <TAB> to see options for the providers.
+
+> Enter provider for API inference: meta-reference
+> Enter provider for API safety: meta-reference
+> Enter provider for API agents: meta-reference
+> Enter provider for API memory: meta-reference
+> Enter provider for API datasetio: meta-reference
+> Enter provider for API scoring: meta-reference
+> Enter provider for API eval: meta-reference
+> Enter provider for API telemetry: meta-reference
+
+ > (Optional) Enter a short description for your Llama Stack:
+
+You can now edit ~/.llama/distributions/llamastack-my-local-stack/my-local-stack-run.yaml and run `llama stack run ~/.llama/distributions/llamastack-my-local-stack/my-local-stack-run.yaml`
 ```
+:::
 
-Running the command above will allow you to fill in the configuration to build your Llama Stack distribution, you will see the following outputs.
+:::{tab-item} Building from a template
-
-```
-> Enter an unique name for identifying your Llama Stack build distribution (e.g. my-local-stack): 8b-instruct
-> Enter the image type you want your distribution to be built with (docker or conda): conda
-
- Llama Stack is composed of several APIs working together. Let's configure the providers (implementations) you want to use for these APIs.
-> Enter the API provider for the inference API: (default=meta-reference): meta-reference
-> Enter the API provider for the safety API: (default=meta-reference): meta-reference
-> Enter the API provider for the agents API: (default=meta-reference): meta-reference
-> Enter the API provider for the memory API: (default=meta-reference): meta-reference
-> Enter the API provider for the telemetry API: (default=meta-reference): meta-reference
-
- > (Optional) Enter a short description for your Llama Stack distribution:
-
-Build spec configuration saved at ~/.conda/envs/llamastack-my-local-llama-stack/8b-instruct-build.yaml
-```
-
-**Ollama (optional)**
-
-If you plan to use Ollama for inference, you'll need to install the server [via these instructions](https://ollama.com/download).
-
-
-#### Building from templates
 - To build from alternative API providers, we provide distribution templates for users to get started building a distribution backed by different providers.
 
 The following command will allow you to see the available templates and their corresponding providers.
@@ -59,17 +62,20 @@ llama stack build --list-templates
 +------------------------------+--------------------------------------------+----------------------------------------------------------------------------------+
 | Template Name                | Providers                                  | Description                                                                      |
 +------------------------------+--------------------------------------------+----------------------------------------------------------------------------------+
-| bedrock                      | {                                          | Use Amazon Bedrock APIs.                                                         |
+| hf-serverless                | {                                          | Like local, but use Hugging Face Inference API (serverless) for running LLM      |
-|                              |   "inference": "remote::bedrock",          |                                                                                  |
+|                              |   "inference": "remote::hf::serverless",   | inference.                                                                       |
-|                              |   "memory": "meta-reference",              |                                                                                  |
+|                              |   "memory": "meta-reference",              | See https://hf.co/docs/api-inference.                                            |
 |                              |   "safety": "meta-reference",              |                                                                                  |
 |                              |   "agents": "meta-reference",              |                                                                                  |
 |                              |   "telemetry": "meta-reference"            |                                                                                  |
 |                              | }                                          |                                                                                  |
 +------------------------------+--------------------------------------------+----------------------------------------------------------------------------------+
-| databricks                   | {                                          | Use Databricks for running LLM inference                                         |
+| together                     | {                                          | Use Together.ai for running LLM inference                                        |
-|                              |   "inference": "remote::databricks",       |                                                                                  |
+|                              |   "inference": "remote::together",         |                                                                                  |
-|                              |   "memory": "meta-reference",              |                                                                                  |
+|                              |   "memory": [                              |                                                                                  |
+|                              |     "meta-reference",                      |                                                                                  |
+|                              |     "remote::weaviate"                     |                                                                                  |
+|                              |   ],                                       |                                                                                  |
 |                              |   "safety": "meta-reference",              |                                                                                  |
 |                              |   "agents": "meta-reference",              |                                                                                  |
 |                              |   "telemetry": "meta-reference"            |                                                                                  |
@@ -88,17 +94,37 @@ llama stack build --list-templates
 |                              |   "telemetry": "meta-reference"            |                                                                                  |
 |                              | }                                          |                                                                                  |
 +------------------------------+--------------------------------------------+----------------------------------------------------------------------------------+
-| hf-endpoint                  | {                                          | Like local, but use Hugging Face Inference Endpoints for running LLM inference.  |
+| databricks                   | {                                          | Use Databricks for running LLM inference                                         |
-|                              |   "inference": "remote::hf::endpoint",     | See https://hf.co/docs/api-endpoints.                                            |
+|                              |   "inference": "remote::databricks",       |                                                                                  |
 |                              |   "memory": "meta-reference",              |                                                                                  |
 |                              |   "safety": "meta-reference",              |                                                                                  |
 |                              |   "agents": "meta-reference",              |                                                                                  |
 |                              |   "telemetry": "meta-reference"            |                                                                                  |
 |                              | }                                          |                                                                                  |
 +------------------------------+--------------------------------------------+----------------------------------------------------------------------------------+
-| hf-serverless                | {                                          | Like local, but use Hugging Face Inference API (serverless) for running LLM      |
+| vllm                         | {                                          | Like local, but use vLLM for running LLM inference                               |
-|                              |   "inference": "remote::hf::serverless",   | inference.                                                                       |
+|                              |   "inference": "vllm",                     |                                                                                  |
-|                              |   "memory": "meta-reference",              | See https://hf.co/docs/api-inference.                                            |
+|                              |   "memory": "meta-reference",              |                                                                                  |
+|                              |   "safety": "meta-reference",              |                                                                                  |
+|                              |   "agents": "meta-reference",              |                                                                                  |
+|                              |   "telemetry": "meta-reference"            |                                                                                  |
+|                              | }                                          |                                                                                  |
++------------------------------+--------------------------------------------+----------------------------------------------------------------------------------+
+| tgi                          | {                                          | Use TGI for running LLM inference                                                |
+|                              |   "inference": "remote::tgi",              |                                                                                  |
+|                              |   "memory": [                              |                                                                                  |
+|                              |     "meta-reference",                      |                                                                                  |
+|                              |     "remote::chromadb",                    |                                                                                  |
+|                              |     "remote::pgvector"                     |                                                                                  |
+|                              |   ],                                       |                                                                                  |
+|                              |   "safety": "meta-reference",              |                                                                                  |
+|                              |   "agents": "meta-reference",              |                                                                                  |
+|                              |   "telemetry": "meta-reference"            |                                                                                  |
+|                              | }                                          |                                                                                  |
++------------------------------+--------------------------------------------+----------------------------------------------------------------------------------+
+| bedrock                      | {                                          | Use Amazon Bedrock APIs.                                                         |
+|                              |   "inference": "remote::bedrock",          |                                                                                  |
+|                              |   "memory": "meta-reference",              |                                                                                  |
 |                              |   "safety": "meta-reference",              |                                                                                  |
 |                              |   "agents": "meta-reference",              |                                                                                  |
 |                              |   "telemetry": "meta-reference"            |                                                                                  |
@@ -140,31 +166,8 @@ llama stack build --list-templates
 |                              |   "telemetry": "meta-reference"            |                                                                                  |
 |                              | }                                          |                                                                                  |
 +------------------------------+--------------------------------------------+----------------------------------------------------------------------------------+
-| tgi                          | {                                          | Use TGI for running LLM inference                                                |
+| hf-endpoint                  | {                                          | Like local, but use Hugging Face Inference Endpoints for running LLM inference.  |
-|                              |   "inference": "remote::tgi",              |                                                                                  |
+|                              |   "inference": "remote::hf::endpoint",     | See https://hf.co/docs/api-endpoints.                                            |
-|                              |   "memory": [                              |                                                                                  |
-|                              |     "meta-reference",                      |                                                                                  |
-|                              |     "remote::chromadb",                    |                                                                                  |
-|                              |     "remote::pgvector"                     |                                                                                  |
-|                              |   ],                                       |                                                                                  |
-|                              |   "safety": "meta-reference",              |                                                                                  |
-|                              |   "agents": "meta-reference",              |                                                                                  |
-|                              |   "telemetry": "meta-reference"            |                                                                                  |
-|                              | }                                          |                                                                                  |
-+------------------------------+--------------------------------------------+----------------------------------------------------------------------------------+
-| together                     | {                                          | Use Together.ai for running LLM inference                                        |
-|                              |   "inference": "remote::together",         |                                                                                  |
-|                              |   "memory": [                              |                                                                                  |
-|                              |     "meta-reference",                      |                                                                                  |
-|                              |     "remote::weaviate"                     |                                                                                  |
-|                              |   ],                                       |                                                                                  |
-|                              |   "safety": "remote::together",            |                                                                                  |
-|                              |   "agents": "meta-reference",              |                                                                                  |
-|                              |   "telemetry": "meta-reference"            |                                                                                  |
-|                              | }                                          |                                                                                  |
-+------------------------------+--------------------------------------------+----------------------------------------------------------------------------------+
-| vllm                         | {                                          | Like local, but use vLLM for running LLM inference                               |
-|                              |   "inference": "vllm",                     |                                                                                  |
 |                              |   "memory": "meta-reference",              |                                                                                  |
 |                              |   "safety": "meta-reference",              |                                                                                  |
 |                              |   "agents": "meta-reference",              |                                                                                  |
@@ -175,6 +178,7 @@ llama stack build --list-templates
 
 You may then pick a template to build your distribution with providers fitted to your liking.
 
+For example, to build a distribution with TGI as the inference provider, you can run:
 ```
 llama stack build --template tgi
 ```
@@ -182,15 +186,14 @@ llama stack build --template tgi
 ```
 $ llama stack build --template tgi
 ...
-...
+You can now edit ~/.llama/distributions/llamastack-tgi/tgi-run.yaml and run `llama stack run ~/.llama/distributions/llamastack-tgi/tgi-run.yaml`
-Build spec configuration saved at ~/.conda/envs/llamastack-tgi/tgi-build.yaml
-You may now run `llama stack configure tgi` or `llama stack configure ~/.conda/envs/llamastack-tgi/tgi-build.yaml`
 ```
+:::
 
-#### Building from config file
+:::{tab-item} Building from a pre-existing build config file
 - In addition to templates, you may customize the build to your liking through editing config files and build from config files with the following command.
 
-- The config file will be of contents like the ones in `llama_stack/distributions/templates/`.
+- The config file will be of contents like the ones in `llama_stack/templates/*build.yaml`.
 
 ```
 $ cat llama_stack/templates/ollama/build.yaml
@@ -210,148 +213,111 @@ image_type: conda
 ```
 llama stack build --config llama_stack/templates/ollama/build.yaml
 ```
+:::
 
-#### How to build distribution with Docker image
+:::{tab-item} Building Docker
-
 > [!TIP]
 > Podman is supported as an alternative to Docker. Set `DOCKER_BINARY` to `podman` in your environment to use Podman.
 
 To build a docker image, you may start off from a template and use the `--image-type docker` flag to specify `docker` as the build image type.
 
 ```
-llama stack build --template local --image-type docker
+llama stack build --template ollama --image-type docker
 ```
 
-Alternatively, you may use a config file and set `image_type` to `docker` in our `<name>-build.yaml` file, and run `llama stack build <name>-build.yaml`. The `<name>-build.yaml` will be of contents like:
-
 ```
-name: local-docker-example
+$ llama stack build --template ollama --image-type docker
-distribution_spec:
-  description: Use code from `llama_stack` itself to serve all llama stack APIs
-  docker_image: null
-  providers:
-    inference: meta-reference
-    memory: meta-reference-faiss
-    safety: meta-reference
-    agentic_system: meta-reference
-    telemetry: console
-image_type: docker
-```
-
-The following command allows you to build a Docker image with the name `<name>`
-```
-llama stack build --config <name>-build.yaml
-
-Dockerfile created successfully in /tmp/tmp.I0ifS2c46A/DockerfileFROM python:3.10-slim
-WORKDIR /app
 ...
+Dockerfile created successfully in /tmp/tmp.viA3a3Rdsg/DockerfileFROM python:3.10-slim
 ...
-You can run it with: podman run -p 8000:8000 llamastack-docker-local
+
-Build spec configuration saved at ~/.llama/distributions/docker/docker-local-build.yaml
+You can now edit ~/meta-llama/llama-stack/tmp/configs/ollama-run.yaml and run `llama stack run ~/meta-llama/llama-stack/tmp/configs/ollama-run.yaml`
 ```
 
+After this step is successful, you should be able to find the built docker image and test it with `llama stack run <path/to/run.yaml>`.
+:::
 
-## Step 2. Configure
+::::
-After our distribution is built (either in form of docker or conda environment), we will run the following command to
+
-```
+
-llama stack configure [ <docker-image-name> | <path/to/name.build.yaml>]
+## Step 2. Run
-```
+Now, let's start the Llama Stack Distribution Server. You will need the YAML configuration file which was written out at the end by the `llama stack build` step.
-- For `conda` environments: <path/to/name.build.yaml> would be the generated build spec saved from Step 1.
-- For `docker` images downloaded from Dockerhub, you could also use <docker-image-name> as the argument.
-   - Run `docker images` to check list of available images on your machine.
 
 ```
-$ llama stack configure tgi
+llama stack run ~/.llama/distributions/llamastack-my-local-stack/my-local-stack-run.yaml
-
-Configuring API: inference (meta-reference)
-Enter value for model (existing: Meta-Llama3.1-8B-Instruct) (required):
-Enter value for quantization (optional):
-Enter value for torch_seed (optional):
-Enter value for max_seq_len (existing: 4096) (required):
-Enter value for max_batch_size (existing: 1) (required):
-
-Configuring API: memory (meta-reference-faiss)
-
-Configuring API: safety (meta-reference)
-Do you want to configure llama_guard_shield? (y/n): y
-Entering sub-configuration for llama_guard_shield:
-Enter value for model (default: Llama-Guard-3-1B) (required):
-Enter value for excluded_categories (default: []) (required):
-Enter value for disable_input_check (default: False) (required):
-Enter value for disable_output_check (default: False) (required):
-Do you want to configure prompt_guard_shield? (y/n): y
-Entering sub-configuration for prompt_guard_shield:
-Enter value for model (default: Prompt-Guard-86M) (required):
-
-Configuring API: agentic_system (meta-reference)
-Enter value for brave_search_api_key (optional):
-Enter value for bing_search_api_key (optional):
-Enter value for wolfram_api_key (optional):
-
-Configuring API: telemetry (console)
-
-YAML configuration has been written to ~/.llama/builds/conda/tgi-run.yaml
 ```
 
-After this step is successful, you should be able to find a run configuration spec in `~/.llama/builds/conda/tgi-run.yaml` with the following contents. You may edit this file to change the settings.
-
-As you can see, we did basic configuration above and configured:
-- inference to run on model `Meta-Llama3.1-8B-Instruct` (obtained from `llama model list`)
-- Llama Guard safety shield with model `Llama-Guard-3-1B`
-- Prompt Guard safety shield with model `Prompt-Guard-86M`
-
-For how these configurations are stored as yaml, checkout the file printed at the end of the configuration.
-
-Note that all configurations as well as models are stored in `~/.llama`
-
-
-## Step 3. Run
-Now, let's start the Llama Stack Distribution Server. You will need the YAML configuration file which was written out at the end by the `llama stack configure` step.
-
-```
-llama stack run 8b-instruct
 ```
+$ llama stack run ~/.llama/distributions/llamastack-my-local-stack/my-local-stack-run.yaml
 
-You should see the Llama Stack server start and print the APIs that it is supporting
+Loaded model...
+Serving API datasets
+ GET /datasets/get
+ GET /datasets/list
+ POST /datasets/register
+Serving API inspect
+ GET /health
+ GET /providers/list
+ GET /routes/list
+Serving API inference
+ POST /inference/chat_completion
+ POST /inference/completion
+ POST /inference/embeddings
+Serving API scoring_functions
+ GET /scoring_functions/get
+ GET /scoring_functions/list
+ POST /scoring_functions/register
+Serving API scoring
+ POST /scoring/score
+ POST /scoring/score_batch
+Serving API memory_banks
+ GET /memory_banks/get
+ GET /memory_banks/list
+ POST /memory_banks/register
+Serving API memory
+ POST /memory/insert
+ POST /memory/query
+Serving API safety
+ POST /safety/run_shield
+Serving API eval
+ POST /eval/evaluate
+ POST /eval/evaluate_batch
+ POST /eval/job/cancel
+ GET /eval/job/result
+ GET /eval/job/status
+Serving API shields
+ GET /shields/get
+ GET /shields/list
+ POST /shields/register
+Serving API datasetio
+ GET /datasetio/get_rows_paginated
+Serving API telemetry
+ GET /telemetry/get_trace
+ POST /telemetry/log_event
+Serving API models
+ GET /models/get
+ GET /models/list
+ POST /models/register
+Serving API agents
+ POST /agents/create
+ POST /agents/session/create
+ POST /agents/turn/create
+ POST /agents/delete
+ POST /agents/session/delete
+ POST /agents/session/get
+ POST /agents/step/get
+ POST /agents/turn/get
 
-```
+Listening on ['::', '0.0.0.0']:5000
-$ llama stack run 8b-instruct
+INFO:     Started server process [2935911]
-
-> initializing model parallel with size 1
-> initializing ddp with size 1
-> initializing pipeline with size 1
-Loaded in 19.28 seconds
-NCCL version 2.20.5+cuda12.4
-Finished model load YES READY
-Serving POST /inference/batch_chat_completion
-Serving POST /inference/batch_completion
-Serving POST /inference/chat_completion
-Serving POST /inference/completion
-Serving POST /safety/run_shield
-Serving POST /agentic_system/memory_bank/attach
-Serving POST /agentic_system/create
-Serving POST /agentic_system/session/create
-Serving POST /agentic_system/turn/create
-Serving POST /agentic_system/delete
-Serving POST /agentic_system/session/delete
-Serving POST /agentic_system/memory_bank/detach
-Serving POST /agentic_system/session/get
-Serving POST /agentic_system/step/get
-Serving POST /agentic_system/turn/get
-Listening on :::5000
-INFO:     Started server process [453333]
 INFO:     Waiting for application startup.
 INFO:     Application startup complete.
-INFO:     Uvicorn running on http://[::]:5000 (Press CTRL+C to quit)
+INFO:     Uvicorn running on http://['::', '0.0.0.0']:5000 (Press CTRL+C to quit)
+INFO:     2401:db00:35c:2d2b:face:0:c9:0:54678 - "GET /models/list HTTP/1.1" 200 OK
 ```
 
-> [!NOTE]
-> Configuration is in `~/.llama/builds/local/conda/tgi-run.yaml`. Feel free to increase `max_seq_len`.
-
 > [!IMPORTANT]
 > The "local" distribution inference server currently only supports CUDA. It will not work on Apple Silicon machines.
 
 > [!TIP]
 > You might need to use the flag `--disable-ipv6` to  Disable IPv6 support
-
-This server is running a Llama model locally.

docs/source/getting_started/distributions/ondevice_distro/ios_sdk.md

@@ -3,7 +3,7 @@
 We offer both remote and on-device use of Llama Stack in Swift via two components:
 
 1. [llama-stack-client-swift](https://github.com/meta-llama/llama-stack-client-swift/)
-2. [LocalInferenceImpl](https://github.com/meta-llama/llama-stack/tree/main/llama_stack/providers/impls/ios/inference)
+2. [LocalInferenceImpl](https://github.com/meta-llama/llama-stack/tree/main/llama_stack/providers/inline/ios/inference)
 
 ```{image} ../../../../_static/remote_or_local.gif
 :alt: Seamlessly switching between local, on-device inference and remote hosted inference

docs/source/getting_started/distributions/remote_hosted_distro/bedrock.md

@@ -0,0 +1,58 @@
+# Bedrock Distribution
+
+### Connect to a Llama Stack Bedrock Endpoint
+- You may connect to Amazon Bedrock APIs for running LLM inference
+
+The `llamastack/distribution-bedrock` distribution consists of the following provider configurations.
+
+
+| **API**         	| **Inference** 	| **Agents**     	| **Memory**     	| **Safety**     	| **Telemetry**  	|
+|-----------------	|---------------	|----------------	|----------------	|----------------	|----------------	|
+| **Provider(s)** 	| remote::bedrock | meta-reference 	| meta-reference 	| remote::bedrock | meta-reference 	|
+
+
+### Docker: Start the Distribution (Single Node CPU)
+
+> [!NOTE]
+> This assumes you have valid AWS credentials configured with access to Amazon Bedrock.
+
+```
+$ cd distributions/bedrock && docker compose up
+```
+
+Make sure in your `run.yaml` file, your inference provider is pointing to the correct AWS configuration. E.g.
+```
+inference:
+  - provider_id: bedrock0
+    provider_type: remote::bedrock
+    config:
+      aws_access_key_id: <AWS_ACCESS_KEY_ID>
+      aws_secret_access_key: <AWS_SECRET_ACCESS_KEY>
+      aws_session_token: <AWS_SESSION_TOKEN>
+      region_name: <AWS_REGION>
+```
+
+### Conda llama stack run (Single Node CPU)
+
+```bash
+llama stack build --template bedrock --image-type conda
+# -- modify run.yaml with valid AWS credentials
+llama stack run ./run.yaml
+```
+
+### (Optional) Update Model Serving Configuration
+
+Use `llama-stack-client models list` to check the available models served by Amazon Bedrock.
+
+```
+$ llama-stack-client models list
++------------------------------+------------------------------+---------------+------------+
+| identifier                   | llama_model                  | provider_id   | metadata   |
++==============================+==============================+===============+============+
+| Llama3.1-8B-Instruct         | meta.llama3-1-8b-instruct-v1:0 | bedrock0     | {}         |
++------------------------------+------------------------------+---------------+------------+
+| Llama3.1-70B-Instruct        | meta.llama3-1-70b-instruct-v1:0 | bedrock0     | {}         |
++------------------------------+------------------------------+---------------+------------+
+| Llama3.1-405B-Instruct       | meta.llama3-1-405b-instruct-v1:0 | bedrock0     | {}         |
++------------------------------+------------------------------+---------------+------------+
+```

docs/source/getting_started/distributions/self_hosted_distro/ollama.md

@@ -102,7 +102,7 @@ ollama pull llama3.1:70b-instruct-fp16
 ```
 
 > [!NOTE]
-> Please check the [OLLAMA_SUPPORTED_MODELS](https://github.com/meta-llama/llama-stack/blob/main/llama_stack/providers/adapters/inference/ollama/ollama.py) for the supported Ollama models.
+> Please check the [OLLAMA_SUPPORTED_MODELS](https://github.com/meta-llama/llama-stack/blob/main/llama_stack/providers.remote/inference/ollama/ollama.py) for the supported Ollama models.
 
 
 To serve a new model with `ollama`

docs/source/getting_started/index.md

@@ -386,7 +386,7 @@ ollama pull llama3.1:8b-instruct-fp16
 ollama pull llama3.1:70b-instruct-fp16
 ```
 
-> Please check the [OLLAMA_SUPPORTED_MODELS](https://github.com/meta-llama/llama-stack/blob/main/llama_stack/providers/adapters/inference/ollama/ollama.py) for the supported Ollama models.
+> Please check the [OLLAMA_SUPPORTED_MODELS](https://github.com/meta-llama/llama-stack/blob/main/llama_stack/providers.remote/inference/ollama/ollama.py) for the supported Ollama models.
 
 
 To serve a new model with `ollama`

llama_stack/apis/safety/safety.py

@@ -39,7 +39,7 @@ class RunShieldResponse(BaseModel):
 
 
 class ShieldStore(Protocol):
-    def get_shield(self, identifier: str) -> ShieldDef: ...
+    async def get_shield(self, identifier: str) -> ShieldDef: ...
 
 
 @runtime_checkable
@@ -48,5 +48,5 @@ class Safety(Protocol):
 
     @webmethod(route="/safety/run_shield")
     async def run_shield(
-        self, shield_type: str, messages: List[Message], params: Dict[str, Any] = None
+        self, identifier: str, messages: List[Message], params: Dict[str, Any] = None
     ) -> RunShieldResponse: ...

llama_stack/apis/shields/shields.py

@@ -46,7 +46,7 @@ class Shields(Protocol):
     async def list_shields(self) -> List[ShieldDefWithProvider]: ...
 
     @webmethod(route="/shields/get", method="GET")
-    async def get_shield(self, shield_type: str) -> Optional[ShieldDefWithProvider]: ...
+    async def get_shield(self, identifier: str) -> Optional[ShieldDefWithProvider]: ...
 
     @webmethod(route="/shields/register", method="POST")
     async def register_shield(self, shield: ShieldDefWithProvider) -> None: ...

llama_stack/cli/stack/build.py

@@ -12,6 +12,10 @@ import os
 from functools import lru_cache
 from pathlib import Path
 
+from llama_stack.distribution.distribution import get_provider_registry
+from llama_stack.distribution.utils.dynamic import instantiate_class_type
+
+
 TEMPLATES_PATH = Path(os.path.relpath(__file__)).parent.parent.parent / "templates"
 
 
@@ -176,6 +180,66 @@ class StackBuild(Subcommand):
                 return
             self._run_stack_build_command_from_build_config(build_config)
 
+    def _generate_run_config(self, build_config: BuildConfig, build_dir: Path) -> None:
+        """
+        Generate a run.yaml template file for user to edit from a build.yaml file
+        """
+        import json
+
+        import yaml
+        from termcolor import cprint
+
+        from llama_stack.distribution.build import ImageType
+
+        apis = list(build_config.distribution_spec.providers.keys())
+        run_config = StackRunConfig(
+            built_at=datetime.now(),
+            docker_image=(
+                build_config.name
+                if build_config.image_type == ImageType.docker.value
+                else None
+            ),
+            image_name=build_config.name,
+            conda_env=(
+                build_config.name
+                if build_config.image_type == ImageType.conda.value
+                else None
+            ),
+            apis=apis,
+            providers={},
+        )
+        # build providers dict
+        provider_registry = get_provider_registry()
+        for api in apis:
+            run_config.providers[api] = []
+            provider_types = build_config.distribution_spec.providers[api]
+            if isinstance(provider_types, str):
+                provider_types = [provider_types]
+
+            for i, provider_type in enumerate(provider_types):
+                p_spec = Provider(
+                    provider_id=f"{provider_type}-{i}",
+                    provider_type=provider_type,
+                    config={},
+                )
+                config_type = instantiate_class_type(
+                    provider_registry[Api(api)][provider_type].config_class
+                )
+                p_spec.config = config_type()
+                run_config.providers[api].append(p_spec)
+
+        os.makedirs(build_dir, exist_ok=True)
+        run_config_file = build_dir / f"{build_config.name}-run.yaml"
+
+        with open(run_config_file, "w") as f:
+            to_write = json.loads(run_config.model_dump_json())
+            f.write(yaml.dump(to_write, sort_keys=False))
+
+        cprint(
+            f"You can now edit {run_config_file} and run `llama stack run {run_config_file}`",
+            color="green",
+        )
+
     def _run_stack_build_command_from_build_config(
         self, build_config: BuildConfig
     ) -> None:
@@ -183,48 +247,24 @@ class StackBuild(Subcommand):
         import os
 
         import yaml
-        from termcolor import cprint
 
-        from llama_stack.distribution.build import build_image, ImageType
+        from llama_stack.distribution.build import build_image
         from llama_stack.distribution.utils.config_dirs import DISTRIBS_BASE_DIR
-        from llama_stack.distribution.utils.serialize import EnumEncoder
 
         # save build.yaml spec for building same distribution again
-        if build_config.image_type == ImageType.docker.value:
+        build_dir = DISTRIBS_BASE_DIR / f"llamastack-{build_config.name}"
-            # docker needs build file to be in the llama-stack repo dir to be able to copy over to the image
-            llama_stack_path = Path(
-                os.path.abspath(__file__)
-            ).parent.parent.parent.parent
-            build_dir = llama_stack_path / "tmp/configs/"
-        else:
-            build_dir = DISTRIBS_BASE_DIR / f"llamastack-{build_config.name}"
-
         os.makedirs(build_dir, exist_ok=True)
         build_file_path = build_dir / f"{build_config.name}-build.yaml"
 
         with open(build_file_path, "w") as f:
-            to_write = json.loads(json.dumps(build_config.dict(), cls=EnumEncoder))
+            to_write = json.loads(build_config.model_dump_json())
             f.write(yaml.dump(to_write, sort_keys=False))
 
         return_code = build_image(build_config, build_file_path)
         if return_code != 0:
             return
 
-        configure_name = (
+        self._generate_run_config(build_config, build_dir)
-            build_config.name
-            if build_config.image_type == "conda"
-            else (f"llamastack-{build_config.name}")
-        )
-        if build_config.image_type == "conda":
-            cprint(
-                f"You can now run `llama stack configure {configure_name}`",
-                color="green",
-            )
-        else:
-            cprint(
-                f"You can now edit your run.yaml file and run `docker run -it -p 5000:5000 {build_config.name}`. See full command in llama-stack/distributions/",
-                color="green",
-            )
 
     def _run_template_list_cmd(self, args: argparse.Namespace) -> None:
         import json

llama_stack/cli/stack/configure.py

@@ -7,8 +7,6 @@
 import argparse
 
 from llama_stack.cli.subcommand import Subcommand
-from llama_stack.distribution.utils.config_dirs import BUILDS_BASE_DIR
-from llama_stack.distribution.datatypes import *  # noqa: F403
 
 
 class StackConfigure(Subcommand):
@@ -39,123 +37,10 @@ class StackConfigure(Subcommand):
         )
 
     def _run_stack_configure_cmd(self, args: argparse.Namespace) -> None:
-        import json
+        self.parser.error(
-        import os
+            """
-        import subprocess
+            DEPRECATED! llama stack configure has been deprecated.
-        from pathlib import Path
+            Please use llama stack run --config <path/to/run.yaml> instead.
-
+            Please see example run.yaml in /distributions folder.
-        import pkg_resources
+            """
-
-        import yaml
-        from termcolor import cprint
-
-        from llama_stack.distribution.build import ImageType
-        from llama_stack.distribution.utils.exec import run_with_pty
-
-        docker_image = None
-
-        build_config_file = Path(args.config)
-        if build_config_file.exists():
-            with open(build_config_file, "r") as f:
-                build_config = BuildConfig(**yaml.safe_load(f))
-                self._configure_llama_distribution(build_config, args.output_dir)
-            return
-
-        conda_dir = (
-            Path(os.path.expanduser("~/.conda/envs")) / f"llamastack-{args.config}"
-        )
-        output = subprocess.check_output(["bash", "-c", "conda info --json"])
-        conda_envs = json.loads(output.decode("utf-8"))["envs"]
-
-        for x in conda_envs:
-            if x.endswith(f"/llamastack-{args.config}"):
-                conda_dir = Path(x)
-                break
-
-        build_config_file = Path(conda_dir) / f"{args.config}-build.yaml"
-        if build_config_file.exists():
-            with open(build_config_file, "r") as f:
-                build_config = BuildConfig(**yaml.safe_load(f))
-
-            cprint(f"Using {build_config_file}...", "green")
-            self._configure_llama_distribution(build_config, args.output_dir)
-            return
-
-        docker_image = args.config
-        builds_dir = BUILDS_BASE_DIR / ImageType.docker.value
-        if args.output_dir:
-            builds_dir = Path(output_dir)
-        os.makedirs(builds_dir, exist_ok=True)
-
-        script = pkg_resources.resource_filename(
-            "llama_stack", "distribution/configure_container.sh"
-        )
-        script_args = [script, docker_image, str(builds_dir)]
-
-        return_code = run_with_pty(script_args)
-        if return_code != 0:
-            self.parser.error(
-                f"Failed to configure container {docker_image} with return code {return_code}. Please run `llama stack build` first. "
-            )
-
-    def _configure_llama_distribution(
-        self,
-        build_config: BuildConfig,
-        output_dir: Optional[str] = None,
-    ):
-        import json
-        import os
-        from pathlib import Path
-
-        import yaml
-        from termcolor import cprint
-
-        from llama_stack.distribution.configure import (
-            configure_api_providers,
-            parse_and_maybe_upgrade_config,
-        )
-        from llama_stack.distribution.utils.serialize import EnumEncoder
-
-        builds_dir = BUILDS_BASE_DIR / build_config.image_type
-        if output_dir:
-            builds_dir = Path(output_dir)
-        os.makedirs(builds_dir, exist_ok=True)
-        image_name = build_config.name.replace("::", "-")
-        run_config_file = builds_dir / f"{image_name}-run.yaml"
-
-        if run_config_file.exists():
-            cprint(
-                f"Configuration already exists at `{str(run_config_file)}`. Will overwrite...",
-                "yellow",
-                attrs=["bold"],
-            )
-            config_dict = yaml.safe_load(run_config_file.read_text())
-            config = parse_and_maybe_upgrade_config(config_dict)
-        else:
-            config = StackRunConfig(
-                built_at=datetime.now(),
-                image_name=image_name,
-                apis=list(build_config.distribution_spec.providers.keys()),
-                providers={},
-            )
-
-        config = configure_api_providers(config, build_config.distribution_spec)
-
-        config.docker_image = (
-            image_name if build_config.image_type == "docker" else None
-        )
-        config.conda_env = image_name if build_config.image_type == "conda" else None
-
-        with open(run_config_file, "w") as f:
-            to_write = json.loads(json.dumps(config.dict(), cls=EnumEncoder))
-            f.write(yaml.dump(to_write, sort_keys=False))
-
-        cprint(
-            f"> YAML configuration has been written to `{run_config_file}`.",
-            color="blue",
-        )
-
-        cprint(
-            f"You can now run `llama stack run {image_name} --port PORT`",
-            color="green",
         )

llama_stack/cli/stack/run.py

@@ -45,7 +45,6 @@ class StackRun(Subcommand):
 
         import pkg_resources
         import yaml
-        from termcolor import cprint
 
         from llama_stack.distribution.build import ImageType
         from llama_stack.distribution.configure import parse_and_maybe_upgrade_config
@@ -71,14 +70,12 @@ class StackRun(Subcommand):
 
         if not config_file.exists():
             self.parser.error(
-                f"File {str(config_file)} does not exist. Please run `llama stack build` and `llama stack configure <name>` to generate a run.yaml file"
+                f"File {str(config_file)} does not exist. Please run `llama stack build` to generate (and optionally edit) a run.yaml file"
             )
             return
 
-        cprint(f"Using config `{config_file}`", "green")
+        config_dict = yaml.safe_load(config_file.read_text())
-        with open(config_file, "r") as f:
+        config = parse_and_maybe_upgrade_config(config_dict)
-            config_dict = yaml.safe_load(config_file.read_text())
-            config = parse_and_maybe_upgrade_config(config_dict)
 
         if config.docker_image:
             script = pkg_resources.resource_filename(

llama_stack/distribution/build_container.sh

@@ -36,7 +36,6 @@ SCRIPT_DIR=$(dirname "$(readlink -f "$0")")
 REPO_DIR=$(dirname $(dirname "$SCRIPT_DIR"))
 DOCKER_BINARY=${DOCKER_BINARY:-docker}
 DOCKER_OPTS=${DOCKER_OPTS:-}
-REPO_CONFIGS_DIR="$REPO_DIR/tmp/configs"
 
 TEMP_DIR=$(mktemp -d)
 
@@ -115,8 +114,6 @@ ENTRYPOINT ["python", "-m", "llama_stack.distribution.server.server"]
 
 EOF
 
-add_to_docker "ADD tmp/configs/$(basename "$build_file_path") ./llamastack-build.yaml"
-
 printf "Dockerfile created successfully in $TEMP_DIR/Dockerfile"
 cat $TEMP_DIR/Dockerfile
 printf "\n"
@@ -138,7 +135,6 @@ set -x
 $DOCKER_BINARY build $DOCKER_OPTS -t $image_name -f "$TEMP_DIR/Dockerfile" "$REPO_DIR" $mounts
 
 # clean up tmp/configs
-rm -rf $REPO_CONFIGS_DIR
 set +x
 
 echo "Success!"

llama_stack/distribution/client.py

@@ -143,14 +143,21 @@ def create_api_client_class(protocol, additional_protocol) -> Type:
             else:
                 data.update(convert(kwargs))
 
-            return dict(
+            ret = dict(
                 method=webmethod.method or "POST",
                 url=url,
-                headers={"Content-Type": "application/json"},
+                headers={
-                params=params,
+                    "Accept": "application/json",
-                json=data,
+                    "Content-Type": "application/json",
+                },
                 timeout=30,
             )
+            if params:
+                ret["params"] = params
+            if data:
+                ret["json"] = data
+
+            return ret
 
     # Add protocol methods to the wrapper
     for p in protocols:

llama_stack/distribution/routers/routers.py

@@ -154,12 +154,12 @@ class SafetyRouter(Safety):
 
     async def run_shield(
         self,
-        shield_type: str,
+        identifier: str,
         messages: List[Message],
         params: Dict[str, Any] = None,
     ) -> RunShieldResponse:
-        return await self.routing_table.get_provider_impl(shield_type).run_shield(
+        return await self.routing_table.get_provider_impl(identifier).run_shield(
-            shield_type=shield_type,
+            identifier=identifier,
             messages=messages,
             params=params,
         )

llama_stack/distribution/routers/routing_tables.py

@@ -182,6 +182,12 @@ class CommonRoutingTableImpl(RoutingTable):
         objs = await self.dist_registry.get_all()
         return [obj for obj in objs if obj.type == type]
 
+    async def get_all_with_types(
+        self, types: List[str]
+    ) -> List[RoutableObjectWithProvider]:
+        objs = await self.dist_registry.get_all()
+        return [obj for obj in objs if obj.type in types]
+
 
 class ModelsRoutingTable(CommonRoutingTableImpl, Models):
     async def list_models(self) -> List[ModelDefWithProvider]:
@@ -198,8 +204,8 @@ class ShieldsRoutingTable(CommonRoutingTableImpl, Shields):
     async def list_shields(self) -> List[ShieldDef]:
         return await self.get_all_with_type("shield")
 
-    async def get_shield(self, shield_type: str) -> Optional[ShieldDefWithProvider]:
+    async def get_shield(self, identifier: str) -> Optional[ShieldDefWithProvider]:
-        return await self.get_object_by_identifier(shield_type)
+        return await self.get_object_by_identifier(identifier)
 
     async def register_shield(self, shield: ShieldDefWithProvider) -> None:
         await self.register_object(shield)
@@ -207,7 +213,14 @@ class ShieldsRoutingTable(CommonRoutingTableImpl, Shields):
 
 class MemoryBanksRoutingTable(CommonRoutingTableImpl, MemoryBanks):
     async def list_memory_banks(self) -> List[MemoryBankDefWithProvider]:
-        return await self.get_all_with_type("memory_bank")
+        return await self.get_all_with_types(
+            [
+                MemoryBankType.vector.value,
+                MemoryBankType.keyvalue.value,
+                MemoryBankType.keyword.value,
+                MemoryBankType.graph.value,
+            ]
+        )
 
     async def get_memory_bank(
         self, identifier: str

llama_stack/distribution/server/server.py

@@ -209,7 +209,8 @@ async def maybe_await(value):
 
 async def sse_generator(event_gen):
     try:
-        async for item in await event_gen:
+        event_gen = await event_gen
+        async for item in event_gen:
             yield create_sse_event(item)
             await asyncio.sleep(0.01)
     except asyncio.CancelledError:
@@ -229,7 +230,6 @@ async def sse_generator(event_gen):
 
 
 def create_dynamic_typed_route(func: Any, method: str):
-
     async def endpoint(request: Request, **kwargs):
         await start_trace(func.__name__)
 

llama_stack/providers/adapters/inference/vllm/__init__.py

@@ -1,15 +0,0 @@
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under the terms described in the LICENSE file in
-# the root directory of this source tree.
-
-from .config import VLLMImplConfig
-from .vllm import VLLMInferenceAdapter
-
-
-async def get_adapter_impl(config: VLLMImplConfig, _deps):
-    assert isinstance(config, VLLMImplConfig), f"Unexpected config type: {type(config)}"
-    impl = VLLMInferenceAdapter(config)
-    await impl.initialize()
-    return impl

llama_stack/providers/adapters/safety/bedrock/config.py

@@ -1,16 +0,0 @@
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under the terms described in the LICENSE file in
-# the root directory of this source tree.
-
-from pydantic import BaseModel, Field
-
-
-class BedrockSafetyConfig(BaseModel):
-    """Configuration information for a guardrail that you want to use in the request."""
-
-    aws_profile: str = Field(
-        default="default",
-        description="The profile on the machine having valid aws credentials. This will ensure separation of creation to invocation",
-    )

llama_stack/providers/adapters/safety/together/config.py

@@ -1,26 +0,0 @@
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under the terms described in the LICENSE file in
-# the root directory of this source tree.
-
-from typing import Optional
-
-from llama_models.schema_utils import json_schema_type
-from pydantic import BaseModel, Field
-
-
-class TogetherProviderDataValidator(BaseModel):
-    together_api_key: str
-
-
-@json_schema_type
-class TogetherSafetyConfig(BaseModel):
-    url: str = Field(
-        default="https://api.together.xyz/v1",
-        description="The URL for the Together AI server",
-    )
-    api_key: Optional[str] = Field(
-        default=None,
-        description="The Together AI API Key (default for the distribution, if any)",
-    )

llama_stack/providers/adapters/safety/together/together.py

@@ -1,101 +0,0 @@
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under the terms described in the LICENSE file in
-# the root directory of this source tree.
-from together import Together
-
-from llama_models.llama3.api.datatypes import *  # noqa: F403
-from llama_stack.apis.safety import *  # noqa: F403
-from llama_stack.distribution.request_headers import NeedsRequestProviderData
-from llama_stack.providers.datatypes import ShieldsProtocolPrivate
-
-from .config import TogetherSafetyConfig
-
-
-TOGETHER_SHIELD_MODEL_MAP = {
-    "llama_guard": "meta-llama/Meta-Llama-Guard-3-8B",
-    "Llama-Guard-3-8B": "meta-llama/Meta-Llama-Guard-3-8B",
-    "Llama-Guard-3-11B-Vision": "meta-llama/Llama-Guard-3-11B-Vision-Turbo",
-}
-
-
-class TogetherSafetyImpl(Safety, NeedsRequestProviderData, ShieldsProtocolPrivate):
-    def __init__(self, config: TogetherSafetyConfig) -> None:
-        self.config = config
-
-    async def initialize(self) -> None:
-        pass
-
-    async def shutdown(self) -> None:
-        pass
-
-    async def register_shield(self, shield: ShieldDef) -> None:
-        raise ValueError("Registering dynamic shields is not supported")
-
-    async def list_shields(self) -> List[ShieldDef]:
-        return [
-            ShieldDef(
-                identifier=ShieldType.llama_guard.value,
-                shield_type=ShieldType.llama_guard.value,
-                params={},
-            )
-        ]
-
-    async def run_shield(
-        self, shield_type: str, messages: List[Message], params: Dict[str, Any] = None
-    ) -> RunShieldResponse:
-        shield_def = await self.shield_store.get_shield(shield_type)
-        if not shield_def:
-            raise ValueError(f"Unknown shield {shield_type}")
-
-        model = shield_def.params.get("model", "llama_guard")
-        if model not in TOGETHER_SHIELD_MODEL_MAP:
-            raise ValueError(f"Unsupported safety model: {model}")
-
-        together_api_key = None
-        if self.config.api_key is not None:
-            together_api_key = self.config.api_key
-        else:
-            provider_data = self.get_request_provider_data()
-            if provider_data is None or not provider_data.together_api_key:
-                raise ValueError(
-                    'Pass Together API Key in the header X-LlamaStack-ProviderData as { "together_api_key": <your api key>}'
-                )
-            together_api_key = provider_data.together_api_key
-
-        # messages can have role assistant or user
-        api_messages = []
-        for message in messages:
-            if message.role in (Role.user.value, Role.assistant.value):
-                api_messages.append({"role": message.role, "content": message.content})
-
-        violation = await get_safety_response(
-            together_api_key, TOGETHER_SHIELD_MODEL_MAP[model], api_messages
-        )
-        return RunShieldResponse(violation=violation)
-
-
-async def get_safety_response(
-    api_key: str, model_name: str, messages: List[Dict[str, str]]
-) -> Optional[SafetyViolation]:
-    client = Together(api_key=api_key)
-    response = client.chat.completions.create(messages=messages, model=model_name)
-    if len(response.choices) == 0:
-        return None
-
-    response_text = response.choices[0].message.content
-    if response_text == "safe":
-        return None
-
-    parts = response_text.split("\n")
-    if len(parts) != 2:
-        return None
-
-    if parts[0] == "unsafe":
-        return SafetyViolation(
-            violation_level=ViolationLevel.ERROR,
-            metadata={"violation_type": parts[1]},
-        )
-
-    return None

llama_stack/providers/datatypes.py

@@ -6,6 +6,7 @@
 
 from enum import Enum
 from typing import Any, List, Optional, Protocol
+from urllib.parse import urlparse
 
 from llama_models.schema_utils import json_schema_type
 from pydantic import BaseModel, Field
@@ -145,11 +146,19 @@ Fully-qualified name of the module to import. The module is expected to have:
 
 class RemoteProviderConfig(BaseModel):
     host: str = "localhost"
-    port: int
+    port: Optional[int] = None
+    protocol: str = "http"
 
     @property
     def url(self) -> str:
-        return f"http://{self.host}:{self.port}"
+        if self.port is None:
+            return f"{self.protocol}://{self.host}"
+        return f"{self.protocol}://{self.host}:{self.port}"
+
+    @classmethod
+    def from_url(cls, url: str) -> "RemoteProviderConfig":
+        parsed = urlparse(url)
+        return cls(host=parsed.hostname, port=parsed.port, protocol=parsed.scheme)
 
 
 @json_schema_type

llama_stack/providers/inline/braintrust/scoring/braintrust.py

@@ -16,7 +16,7 @@ from llama_stack.apis.datasets import *  # noqa: F403
 from autoevals.llm import Factuality
 from autoevals.ragas import AnswerCorrectness
 from llama_stack.providers.datatypes import ScoringFunctionsProtocolPrivate
-from llama_stack.providers.impls.meta_reference.scoring.scoring_fn.common import (
+from llama_stack.providers.inline.meta_reference.scoring.scoring_fn.common import (
     aggregate_average,
 )
 

llama_stack/providers/inline/meta_reference/agents/config.py

@@ -4,10 +4,11 @@
 # This source code is licensed under the terms described in the LICENSE file in
 # the root directory of this source tree.
 
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 
 from llama_stack.providers.utils.kvstore import KVStoreConfig
+from llama_stack.providers.utils.kvstore.config import SqliteKVStoreConfig
 
 
 class MetaReferenceAgentsImplConfig(BaseModel):
-    persistence_store: KVStoreConfig
+    persistence_store: KVStoreConfig = Field(default=SqliteKVStoreConfig())

llama_stack/providers/inline/meta_reference/agents/safety.py

@@ -32,18 +32,18 @@ class ShieldRunnerMixin:
         self.output_shields = output_shields
 
     async def run_multiple_shields(
-        self, messages: List[Message], shield_types: List[str]
+        self, messages: List[Message], identifiers: List[str]
     ) -> None:
         responses = await asyncio.gather(
             *[
                 self.safety_api.run_shield(
-                    shield_type=shield_type,
+                    identifier=identifier,
                     messages=messages,
                 )
-                for shield_type in shield_types
+                for identifier in identifiers
             ]
         )
-        for shield_type, response in zip(shield_types, responses):
+        for identifier, response in zip(identifiers, responses):
             if not response.violation:
                 continue
 
@@ -52,6 +52,6 @@ class ShieldRunnerMixin:
                 raise SafetyException(violation)
             elif violation.violation_level == ViolationLevel.WARN:
                 cprint(
-                    f"[Warn]{shield_type} raised a warning",
+                    f"[Warn]{identifier} raised a warning",
                     color="red",
                 )

llama_stack/providers/inline/meta_reference/agents/tools/safety.py

@@ -9,7 +9,7 @@ from typing import List
 from llama_stack.apis.inference import Message
 from llama_stack.apis.safety import *  # noqa: F403
 
-from llama_stack.providers.impls.meta_reference.agents.safety import ShieldRunnerMixin
+from llama_stack.providers.inline.meta_reference.agents.safety import ShieldRunnerMixin
 
 from .builtin import BaseTool
 

llama_stack/providers/inline/meta_reference/inference/quantization/loader.py

@@ -27,7 +27,7 @@ from torchao.quantization.GPTQ import Int8DynActInt4WeightLinear
 
 from llama_stack.apis.inference import QuantizationType
 
-from llama_stack.providers.impls.meta_reference.inference.config import (
+from llama_stack.providers.inline.meta_reference.inference.config import (
     MetaReferenceQuantizedInferenceConfig,
 )
 

llama_stack/providers/inline/meta_reference/memory/tests/test_faiss.py

@@ -8,9 +8,9 @@ import tempfile
 
 import pytest
 from llama_stack.apis.memory import MemoryBankType, VectorMemoryBankDef
-from llama_stack.providers.impls.meta_reference.memory.config import FaissImplConfig
+from llama_stack.providers.inline.meta_reference.memory.config import FaissImplConfig
 
-from llama_stack.providers.impls.meta_reference.memory.faiss import FaissMemoryImpl
+from llama_stack.providers.inline.meta_reference.memory.faiss import FaissMemoryImpl
 from llama_stack.providers.utils.kvstore.config import SqliteKVStoreConfig
 
 

llama_stack/providers/inline/meta_reference/safety/safety.py

@@ -57,13 +57,13 @@ class MetaReferenceSafetyImpl(Safety, ShieldsProtocolPrivate):
 
     async def run_shield(
         self,
-        shield_type: str,
+        identifier: str,
         messages: List[Message],
         params: Dict[str, Any] = None,
     ) -> RunShieldResponse:
-        shield_def = await self.shield_store.get_shield(shield_type)
+        shield_def = await self.shield_store.get_shield(identifier)
         if not shield_def:
-            raise ValueError(f"Unknown shield {shield_type}")
+            raise ValueError(f"Unknown shield {identifier}")
 
         shield = self.get_shield_impl(shield_def)