mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-07-29 15:23:51 +00:00
Kill --name from llama stack build
This commit is contained in:
Ashwin Bharambe
parent
26d1668f7d
commit
bd424f9a26
5 changed files with 55 additions and 102 deletions
|
|
@ -58,15 +58,15 @@ llama stack build --list-templates
|
|||
You may then pick a template to build your distribution with providers fitted to your liking.
|
||||
|
||||
```
|
||||
llama stack build --template local-tgi --name my-tgi-stack
|
||||
llama stack build --template tgi --name tgi
|
||||
```
|
||||
|
||||
```
|
||||
$ llama stack build --template local-tgi --name my-tgi-stack
|
||||
$ llama stack build --template tgi --name tgi
|
||||
...
|
||||
...
|
||||
Build spec configuration saved at ~/.conda/envs/llamastack-my-tgi-stack/my-tgi-stack-build.yaml
|
||||
You may now run `llama stack configure my-tgi-stack` or `llama stack configure ~/.conda/envs/llamastack-my-tgi-stack/my-tgi-stack-build.yaml`
|
||||
Build spec configuration saved at ~/.conda/envs/llamastack-tgi/tgi-build.yaml
|
||||
You may now run `llama stack configure tgi` or `llama stack configure ~/.conda/envs/llamastack-tgi/tgi-build.yaml`
|
||||
```
|
||||
|
||||
#### Building from config file
|
||||
|
|
@ -75,7 +75,7 @@ You may now run `llama stack configure my-tgi-stack` or `llama stack configure ~
|
|||
- The config file will be of contents like the ones in `llama_stack/distributions/templates/`.
|
||||
|
||||
```
|
||||
$ cat llama_stack/distribution/templates/local-ollama-build.yaml
|
||||
$ cat llama_stack/templates/ollama/build.yaml
|
||||
|
||||
name: local-ollama
|
||||
distribution_spec:
|
||||
|
|
@ -90,7 +90,7 @@ image_type: conda
|
|||
```
|
||||
|
||||
```
|
||||
llama stack build --config llama_stack/distribution/templates/local-ollama-build.yaml
|
||||
llama stack build --config llama_stack/templates/ollama/build.yaml
|
||||
```
|
||||
|
||||
#### How to build distribution with Docker image
|
||||
|
|
@ -101,7 +101,7 @@ llama stack build --config llama_stack/distribution/templates/local-ollama-build
|
|||
To build a docker image, you may start off from a template and use the `--image-type docker` flag to specify `docker` as the build image type.
|
||||
|
||||
```
|
||||
llama stack build --template local --image-type docker --name docker-0
|
||||
llama stack build --template local --image-type docker
|
||||
```
|
||||
|
||||
Alternatively, you may use a config file and set `image_type` to `docker` in our `<name>-build.yaml` file, and run `llama stack build <name>-build.yaml`. The `<name>-build.yaml` will be of contents like:
|
||||
|
|
@ -136,14 +136,14 @@ Build spec configuration saved at ~/.llama/distributions/docker/docker-local-bui
|
|||
## Step 2. Configure
|
||||
After our distribution is built (either in form of docker or conda environment), we will run the following command to
|
||||
```
|
||||
llama stack configure [ <name> | <docker-image-name> | <path/to/name.build.yaml>]
|
||||
llama stack configure [ <docker-image-name> | <path/to/name.build.yaml>]
|
||||
```
|
||||
- For `conda` environments: <path/to/name.build.yaml> would be the generated build spec saved from Step 1.
|
||||
- For `docker` images downloaded from Dockerhub, you could also use <docker-image-name> as the argument.
|
||||
- Run `docker images` to check list of available images on your machine.
|
||||
|
||||
```
|
||||
$ llama stack configure 8b-instruct
|
||||
$ llama stack configure tgi
|
||||
|
||||
Configuring API: inference (meta-reference)
|
||||
Enter value for model (existing: Meta-Llama3.1-8B-Instruct) (required):
|
||||
|
|
@ -172,10 +172,10 @@ Enter value for wolfram_api_key (optional):
|
|||
|
||||
Configuring API: telemetry (console)
|
||||
|
||||
YAML configuration has been written to ~/.llama/builds/conda/8b-instruct-run.yaml
|
||||
YAML configuration has been written to ~/.llama/builds/conda/tgi-run.yaml
|
||||
```
|
||||
|
||||
After this step is successful, you should be able to find a run configuration spec in `~/.llama/builds/conda/8b-instruct-run.yaml` with the following contents. You may edit this file to change the settings.
|
||||
After this step is successful, you should be able to find a run configuration spec in `~/.llama/builds/conda/tgi-run.yaml` with the following contents. You may edit this file to change the settings.
|
||||
|
||||
As you can see, we did basic configuration above and configured:
|
||||
- inference to run on model `Meta-Llama3.1-8B-Instruct` (obtained from `llama model list`)
|
||||
|
|
@ -228,7 +228,7 @@ INFO: Uvicorn running on http://[::]:5000 (Press CTRL+C to quit)
|
|||
```
|
||||
|
||||
> [!NOTE]
|
||||
> Configuration is in `~/.llama/builds/local/conda/8b-instruct-run.yaml`. Feel free to increase `max_seq_len`.
|
||||
> Configuration is in `~/.llama/builds/local/conda/tgi-run.yaml`. Feel free to increase `max_seq_len`.
|
||||
|
||||
> [!IMPORTANT]
|
||||
> The "local" distribution inference server currently only supports CUDA. It will not work on Apple Silicon machines.
|
||||
|
|
|
|||
|
|
@ -229,8 +229,8 @@ You will be shown a Markdown formatted description of the model interface and ho
|
|||
- Please see our [Getting Started](getting_started.md) guide for more details on how to build and start a Llama Stack distribution.
|
||||
|
||||
### Step 3.1 Build
|
||||
In the following steps, imagine we'll be working with a `Llama3.1-8B-Instruct` model. We will name our build `8b-instruct` to help us remember the config. We will start build our distribution (in the form of a Conda environment, or Docker image). In this step, we will specify:
|
||||
- `name`: the name for our distribution (e.g. `8b-instruct`)
|
||||
In the following steps, imagine we'll be working with a `Llama3.1-8B-Instruct` model. We will name our build `tgi` to help us remember the config. We will start build our distribution (in the form of a Conda environment, or Docker image). In this step, we will specify:
|
||||
- `name`: the name for our distribution (e.g. `tgi`)
|
||||
- `image_type`: our build image type (`conda | docker`)
|
||||
- `distribution_spec`: our distribution specs for specifying API providers
|
||||
- `description`: a short description of the configurations for the distribution
|
||||
|
|
@ -279,15 +279,15 @@ llama stack build --list-templates
|
|||
You may then pick a template to build your distribution with providers fitted to your liking.
|
||||
|
||||
```
|
||||
llama stack build --template local-tgi --name my-tgi-stack --image-type conda
|
||||
llama stack build --template tgi --image-type conda
|
||||
```
|
||||
|
||||
```
|
||||
$ llama stack build --template local-tgi --name my-tgi-stack --image-type conda
|
||||
$ llama stack build --template tgi --image-type conda
|
||||
...
|
||||
...
|
||||
Build spec configuration saved at ~/.conda/envs/llamastack-my-tgi-stack/my-tgi-stack-build.yaml
|
||||
You may now run `llama stack configure my-tgi-stack` or `llama stack configure ~/.conda/envs/llamastack-my-tgi-stack/my-tgi-stack-build.yaml`
|
||||
Build spec configuration saved at ~/.conda/envs/llamastack-tgi/tgi-build.yaml
|
||||
You may now run `llama stack configure tgi` or `llama stack configure ~/.conda/envs/llamastack-tgi/tgi-build.yaml`
|
||||
```
|
||||
|
||||
#### Building from config file
|
||||
|
|
@ -319,7 +319,7 @@ llama stack build --config build.yaml
|
|||
To build a docker image, you may start off from a template and use the `--image-type docker` flag to specify `docker` as the build image type.
|
||||
|
||||
```
|
||||
llama stack build --template local --image-type docker --name docker-0
|
||||
llama stack build --template tgi --image-type docker
|
||||
```
|
||||
|
||||
Alternatively, you may use a config file and set `image_type` to `docker` in our `<name>-build.yaml` file, and run `llama stack build <name>-build.yaml`. The `<name>-build.yaml` will be of contents like:
|
||||
|
|
@ -354,14 +354,14 @@ Build spec configuration saved at ~/.llama/distributions/docker/docker-local-bui
|
|||
### Step 3.2 Configure
|
||||
After our distribution is built (either in form of docker or conda environment), we will run the following command to
|
||||
```
|
||||
llama stack configure [ <name> | <docker-image-name> | <path/to/name.build.yaml>]
|
||||
llama stack configure [ <docker-image-name> | <path/to/name-build.yaml>]
|
||||
```
|
||||
- For `conda` environments: <path/to/name.build.yaml> would be the generated build spec saved from Step 1.
|
||||
- For `docker` images downloaded from Dockerhub, you could also use <docker-image-name> as the argument.
|
||||
- Run `docker images` to check list of available images on your machine.
|
||||
|
||||
```
|
||||
$ llama stack configure ~/.llama/distributions/conda/8b-instruct-build.yaml
|
||||
$ llama stack configure ~/.llama/distributions/conda/tgi-build.yaml
|
||||
|
||||
Configuring API: inference (meta-reference)
|
||||
Enter value for model (existing: Llama3.1-8B-Instruct) (required):
|
||||
|
|
@ -409,13 +409,13 @@ Note that all configurations as well as models are stored in `~/.llama`
|
|||
Now, let's start the Llama Stack Distribution Server. You will need the YAML configuration file which was written out at the end by the `llama stack configure` step.
|
||||
|
||||
```
|
||||
llama stack run ~/.llama/builds/conda/8b-instruct-run.yaml
|
||||
llama stack run ~/.llama/builds/conda/tgi-run.yaml
|
||||
```
|
||||
|
||||
You should see the Llama Stack server start and print the APIs that it is supporting
|
||||
|
||||
```
|
||||
$ llama stack run ~/.llama/builds/local/conda/8b-instruct.yaml
|
||||
$ llama stack run ~/.llama/builds/local/conda/tgi-run.yaml
|
||||
|
||||
> initializing model parallel with size 1
|
||||
> initializing ddp with size 1
|
||||
|
|
@ -446,7 +446,7 @@ INFO: Uvicorn running on http://[::]:5000 (Press CTRL+C to quit)
|
|||
```
|
||||
|
||||
> [!NOTE]
|
||||
> Configuration is in `~/.llama/builds/local/conda/8b-instruct-run.yaml`. Feel free to increase `max_seq_len`.
|
||||
> Configuration is in `~/.llama/builds/local/conda/tgi-run.yaml`. Feel free to increase `max_seq_len`.
|
||||
|
||||
> [!IMPORTANT]
|
||||
> The "local" distribution inference server currently only supports CUDA. It will not work on Apple Silicon machines.
|
||||
|
|
|
|||
|
|
@ -279,15 +279,15 @@ llama stack build --list-templates
|
|||
You may then pick a template to build your distribution with providers fitted to your liking.
|
||||
|
||||
```
|
||||
llama stack build --template local-tgi --name my-tgi-stack
|
||||
llama stack build --template tgi
|
||||
```
|
||||
|
||||
```
|
||||
$ llama stack build --template local-tgi --name my-tgi-stack
|
||||
$ llama stack build --template tgi
|
||||
...
|
||||
...
|
||||
Build spec configuration saved at ~/.conda/envs/llamastack-my-tgi-stack/my-tgi-stack-build.yaml
|
||||
You may now run `llama stack configure my-tgi-stack` or `llama stack configure ~/.conda/envs/llamastack-my-tgi-stack/my-tgi-stack-build.yaml`
|
||||
Build spec configuration saved at ~/.conda/envs/llamastack-tgi/tgi-build.yaml
|
||||
You may now run `llama stack configure tgi` or `llama stack configure ~/.conda/envs/llamastack-tgi/tgi-build.yaml`
|
||||
```
|
||||
|
||||
#### Building from config file
|
||||
|
|
@ -319,7 +319,7 @@ llama stack build --config llama_stack/distribution/templates/local-ollama-build
|
|||
To build a docker image, you may start off from a template and use the `--image-type docker` flag to specify `docker` as the build image type.
|
||||
|
||||
```
|
||||
llama stack build --template local --image-type docker --name docker-0
|
||||
llama stack build --template local --image-type docker
|
||||
```
|
||||
|
||||
Alternatively, you may use a config file and set `image_type` to `docker` in our `<name>-build.yaml` file, and run `llama stack build <name>-build.yaml`. The `<name>-build.yaml` will be of contents like:
|
||||
|
|
@ -354,14 +354,14 @@ Build spec configuration saved at ~/.llama/distributions/docker/docker-local-bui
|
|||
### Step 3.2 Configure
|
||||
After our distribution is built (either in form of docker or conda environment), we will run the following command to
|
||||
```
|
||||
llama stack configure [ <name> | <docker-image-name> | <path/to/name.build.yaml>]
|
||||
llama stack configure [ <docker-image-name> | <path/to/name.build.yaml>]
|
||||
```
|
||||
- For `conda` environments: <path/to/name.build.yaml> would be the generated build spec saved from Step 1.
|
||||
- For `docker` images downloaded from Dockerhub, you could also use <docker-image-name> as the argument.
|
||||
- Run `docker images` to check list of available images on your machine.
|
||||
|
||||
```
|
||||
$ llama stack configure ~/.llama/distributions/conda/8b-instruct-build.yaml
|
||||
$ llama stack configure ~/.llama/distributions/conda/tgi-build.yaml
|
||||
|
||||
Configuring API: inference (meta-reference)
|
||||
Enter value for model (existing: Llama3.1-8B-Instruct) (required):
|
||||
|
|
@ -390,10 +390,10 @@ Enter value for wolfram_api_key (optional):
|
|||
|
||||
Configuring API: telemetry (console)
|
||||
|
||||
YAML configuration has been written to ~/.llama/builds/conda/8b-instruct-run.yaml
|
||||
YAML configuration has been written to ~/.llama/builds/conda/tgi-run.yaml
|
||||
```
|
||||
|
||||
After this step is successful, you should be able to find a run configuration spec in `~/.llama/builds/conda/8b-instruct-run.yaml` with the following contents. You may edit this file to change the settings.
|
||||
After this step is successful, you should be able to find a run configuration spec in `~/.llama/builds/conda/tgi-run.yaml` with the following contents. You may edit this file to change the settings.
|
||||
|
||||
As you can see, we did basic configuration above and configured:
|
||||
- inference to run on model `Llama3.1-8B-Instruct` (obtained from `llama model list`)
|
||||
|
|
@ -409,13 +409,13 @@ Note that all configurations as well as models are stored in `~/.llama`
|
|||
Now, let's start the Llama Stack Distribution Server. You will need the YAML configuration file which was written out at the end by the `llama stack configure` step.
|
||||
|
||||
```
|
||||
llama stack run ~/.llama/builds/conda/8b-instruct-run.yaml
|
||||
llama stack run ~/.llama/builds/conda/tgi-run.yaml
|
||||
```
|
||||
|
||||
You should see the Llama Stack server start and print the APIs that it is supporting
|
||||
|
||||
```
|
||||
$ llama stack run ~/.llama/builds/local/conda/8b-instruct.yaml
|
||||
$ llama stack run ~/.llama/builds/conda/tgi-run.yaml
|
||||
|
||||
> initializing model parallel with size 1
|
||||
> initializing ddp with size 1
|
||||
|
|
@ -446,7 +446,7 @@ INFO: Uvicorn running on http://[::]:5000 (Press CTRL+C to quit)
|
|||
```
|
||||
|
||||
> [!NOTE]
|
||||
> Configuration is in `~/.llama/builds/local/conda/8b-instruct-run.yaml`. Feel free to increase `max_seq_len`.
|
||||
> Configuration is in `~/.llama/builds/local/conda/tgi-run.yaml`. Feel free to increase `max_seq_len`.
|
||||
|
||||
> [!IMPORTANT]
|
||||
> The "local" distribution inference server currently only supports CUDA. It will not work on Apple Silicon machines.
|
||||
|
|
|
|||
|
|
@ -218,15 +218,15 @@ llama stack build --list-templates
|
|||
You may then pick a template to build your distribution with providers fitted to your liking.
|
||||
|
||||
```
|
||||
llama stack build --template local-tgi --name my-tgi-stack
|
||||
llama stack build --template tgi
|
||||
```
|
||||
|
||||
```
|
||||
$ llama stack build --template local-tgi --name my-tgi-stack
|
||||
$ llama stack build --template tgi
|
||||
...
|
||||
...
|
||||
Build spec configuration saved at ~/.conda/envs/llamastack-my-tgi-stack/my-tgi-stack-build.yaml
|
||||
You may now run `llama stack configure my-tgi-stack` or `llama stack configure ~/.conda/envs/llamastack-my-tgi-stack/my-tgi-stack-build.yaml`
|
||||
Build spec configuration saved at ~/.conda/envs/llamastack-tgi/tgi-build.yaml
|
||||
You may now run `llama stack configure tgi` or `llama stack configure ~/.conda/envs/llamastack-tgi/tgi-build.yaml`
|
||||
```
|
||||
|
||||
#### Building from config file
|
||||
|
|
@ -250,7 +250,7 @@ image_type: conda
|
|||
```
|
||||
|
||||
```
|
||||
llama stack build --config llama_stack/distribution/templates/local-ollama-build.yaml
|
||||
llama stack build --config llama_stack/templates/ollama/build.yaml
|
||||
```
|
||||
|
||||
#### How to build distribution with Docker image
|
||||
|
|
@ -261,7 +261,7 @@ llama stack build --config llama_stack/distribution/templates/local-ollama-build
|
|||
To build a docker image, you may start off from a template and use the `--image-type docker` flag to specify `docker` as the build image type.
|
||||
|
||||
```
|
||||
llama stack build --template local --image-type docker --name docker-0
|
||||
llama stack build --template tgi --image-type docker
|
||||
```
|
||||
|
||||
Alternatively, you may use a config file and set `image_type` to `docker` in our `<name>-build.yaml` file, and run `llama stack build <name>-build.yaml`. The `<name>-build.yaml` will be of contents like:
|
||||
|
|
@ -296,14 +296,14 @@ Build spec configuration saved at ~/.llama/distributions/docker/docker-local-bui
|
|||
### Step 2. Configure
|
||||
After our distribution is built (either in form of docker or conda environment), we will run the following command to
|
||||
```
|
||||
llama stack configure [ <name> | <docker-image-name> | <path/to/name.build.yaml>]
|
||||
llama stack configure [ <docker-image-name> | <path/to/name.build.yaml>]
|
||||
```
|
||||
- For `conda` environments: <path/to/name.build.yaml> would be the generated build spec saved from Step 1.
|
||||
- For `docker` images downloaded from Dockerhub, you could also use <docker-image-name> as the argument.
|
||||
- Run `docker images` to check list of available images on your machine.
|
||||
|
||||
```
|
||||
$ llama stack configure 8b-instruct
|
||||
$ llama stack configure tgi
|
||||
|
||||
Configuring API: inference (meta-reference)
|
||||
Enter value for model (existing: Meta-Llama3.1-8B-Instruct) (required):
|
||||
|
|
@ -332,10 +332,10 @@ Enter value for wolfram_api_key (optional):
|
|||
|
||||
Configuring API: telemetry (console)
|
||||
|
||||
YAML configuration has been written to ~/.llama/builds/conda/8b-instruct-run.yaml
|
||||
YAML configuration has been written to ~/.llama/builds/conda/tgi-run.yaml
|
||||
```
|
||||
|
||||
After this step is successful, you should be able to find a run configuration spec in `~/.llama/builds/conda/8b-instruct-run.yaml` with the following contents. You may edit this file to change the settings.
|
||||
After this step is successful, you should be able to find a run configuration spec in `~/.llama/builds/conda/tgi-run.yaml` with the following contents. You may edit this file to change the settings.
|
||||
|
||||
As you can see, we did basic configuration above and configured:
|
||||
- inference to run on model `Meta-Llama3.1-8B-Instruct` (obtained from `llama model list`)
|
||||
|
|
@ -351,13 +351,13 @@ Note that all configurations as well as models are stored in `~/.llama`
|
|||
Now, let's start the Llama Stack Distribution Server. You will need the YAML configuration file which was written out at the end by the `llama stack configure` step.
|
||||
|
||||
```
|
||||
llama stack run 8b-instruct
|
||||
llama stack run tgi
|
||||
```
|
||||
|
||||
You should see the Llama Stack server start and print the APIs that it is supporting
|
||||
|
||||
```
|
||||
$ llama stack run 8b-instruct
|
||||
$ llama stack run tgi
|
||||
|
||||
> initializing model parallel with size 1
|
||||
> initializing ddp with size 1
|
||||
|
|
|
|||
|
|
@ -62,12 +62,6 @@ class StackBuild(Subcommand):
|
|||
help="Show the available templates for building a Llama Stack distribution",
|
||||
)
|
||||
|
||||
self.parser.add_argument(
|
||||
"--name",
|
||||
type=str,
|
||||
help="Name of the Llama Stack build to override from template config. This name will be used as paths to store configuration files, build conda environments/docker images. If not specified, will use the name from the template config. ",
|
||||
)
|
||||
|
||||
self.parser.add_argument(
|
||||
"--image-type",
|
||||
type=str,
|
||||
|
|
@ -91,15 +85,9 @@ class StackBuild(Subcommand):
|
|||
return
|
||||
|
||||
if args.template:
|
||||
if not args.name:
|
||||
self.parser.error(
|
||||
"You must specify a name for the build using --name when using a template"
|
||||
)
|
||||
return
|
||||
available_templates = available_templates_specs()
|
||||
for build_config in available_templates:
|
||||
if build_config.name == args.template:
|
||||
build_config.name = args.name
|
||||
if args.image_type:
|
||||
build_config.image_type = args.image_type
|
||||
else:
|
||||
|
|
@ -114,30 +102,14 @@ class StackBuild(Subcommand):
|
|||
)
|
||||
return
|
||||
|
||||
# try to see if we can find a pre-existing build config file through name
|
||||
if args.name:
|
||||
maybe_build_config = self._get_build_config_from_name(args)
|
||||
if maybe_build_config:
|
||||
cprint(
|
||||
f"Building from existing build config for {args.name} in {str(maybe_build_config)}...",
|
||||
"green",
|
||||
)
|
||||
with open(maybe_build_config, "r") as f:
|
||||
build_config = BuildConfig(**yaml.safe_load(f))
|
||||
self._run_stack_build_command_from_build_config(build_config)
|
||||
return
|
||||
|
||||
if not args.config and not args.template:
|
||||
if not args.name:
|
||||
name = prompt(
|
||||
"> Enter a name for your Llama Stack (e.g. my-local-stack): ",
|
||||
validator=Validator.from_callable(
|
||||
lambda x: len(x) > 0,
|
||||
error_message="Name cannot be empty, please enter a name",
|
||||
),
|
||||
)
|
||||
else:
|
||||
name = args.name
|
||||
name = prompt(
|
||||
"> Enter a name for your Llama Stack (e.g. my-local-stack): ",
|
||||
validator=Validator.from_callable(
|
||||
lambda x: len(x) > 0,
|
||||
error_message="Name cannot be empty, please enter a name",
|
||||
),
|
||||
)
|
||||
|
||||
image_type = prompt(
|
||||
"> Enter the image type you want your Llama Stack to be built as (docker or conda): ",
|
||||
|
|
@ -203,25 +175,6 @@ class StackBuild(Subcommand):
|
|||
return
|
||||
self._run_stack_build_command_from_build_config(build_config)
|
||||
|
||||
def _get_build_config_from_name(self, args: argparse.Namespace) -> Optional[Path]:
|
||||
if os.getenv("CONDA_PREFIX", ""):
|
||||
conda_dir = (
|
||||
Path(os.getenv("CONDA_PREFIX")).parent / f"llamastack-{args.name}"
|
||||
)
|
||||
else:
|
||||
cprint(
|
||||
"Cannot find CONDA_PREFIX. Trying default conda path ~/.conda/envs...",
|
||||
color="green",
|
||||
)
|
||||
conda_dir = (
|
||||
Path(os.path.expanduser("~/.conda/envs")) / f"llamastack-{args.name}"
|
||||
)
|
||||
build_config_file = Path(conda_dir) / f"{args.name}-build.yaml"
|
||||
if build_config_file.exists():
|
||||
return build_config_file
|
||||
|
||||
return None
|
||||
|
||||
def _run_stack_build_command_from_build_config(
|
||||
self, build_config: BuildConfig
|
||||
) -> None:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue