diff --git a/.github/workflows/providers-build.yml b/.github/workflows/providers-build.yml index 391acbcf8..53b6edccf 100644 --- a/.github/workflows/providers-build.yml +++ b/.github/workflows/providers-build.yml @@ -112,7 +112,7 @@ jobs: fi entrypoint=$(docker inspect --format '{{ .Config.Entrypoint }}' $IMAGE_ID) echo "Entrypoint: $entrypoint" - if [ "$entrypoint" != "[python -m llama_stack.core.server.server /app/run.yaml]" ]; then + if [ "$entrypoint" != "[llama stack run /app/run.yaml]" ]; then echo "Entrypoint is not correct" exit 1 fi @@ -150,7 +150,7 @@ jobs: fi entrypoint=$(docker inspect --format '{{ .Config.Entrypoint }}' $IMAGE_ID) echo "Entrypoint: $entrypoint" - if [ "$entrypoint" != "[python -m llama_stack.core.server.server /app/run.yaml]" ]; then + if [ "$entrypoint" != "[llama stack run /app/run.yaml]" ]; then echo "Entrypoint is not correct" exit 1 fi diff --git a/docs/docs/concepts/apis/external.mdx b/docs/docs/concepts/apis/external.mdx index 5664e6fa3..42819a4ac 100644 --- a/docs/docs/concepts/apis/external.mdx +++ b/docs/docs/concepts/apis/external.mdx @@ -357,7 +357,7 @@ server: 8. Run the server: ```bash -python -m llama_stack.core.server.server --yaml-config ~/.llama/run-byoa.yaml +llama stack run ~/.llama/run-byoa.yaml ``` 9. Test the API: diff --git a/docs/docs/deploying/kubernetes_deployment.mdx b/docs/docs/deploying/kubernetes_deployment.mdx index a937ce355..8ed1e2756 100644 --- a/docs/docs/deploying/kubernetes_deployment.mdx +++ b/docs/docs/deploying/kubernetes_deployment.mdx @@ -170,7 +170,7 @@ spec: - name: llama-stack image: localhost/llama-stack-run-k8s:latest imagePullPolicy: IfNotPresent - command: ["python", "-m", "llama_stack.core.server.server", "--config", "/app/config.yaml"] + command: ["llama", "stack", "run", "/app/config.yaml"] ports: - containerPort: 5000 volumeMounts: diff --git a/docs/docs/distributions/k8s/stack-k8s.yaml.template b/docs/docs/distributions/k8s/stack-k8s.yaml.template index dfc049f4f..f426b3261 100644 --- a/docs/docs/distributions/k8s/stack-k8s.yaml.template +++ b/docs/docs/distributions/k8s/stack-k8s.yaml.template @@ -52,7 +52,7 @@ spec: value: "${SAFETY_MODEL}" - name: TAVILY_SEARCH_API_KEY value: "${TAVILY_SEARCH_API_KEY}" - command: ["python", "-m", "llama_stack.core.server.server", "/etc/config/stack_run_config.yaml", "--port", "8321"] + command: ["llama", "stack", "run", "/etc/config/stack_run_config.yaml", "--port", "8321"] ports: - containerPort: 8321 volumeMounts: diff --git a/llama_stack/cli/stack/run.py b/llama_stack/cli/stack/run.py index b32b8b3ae..eef17a669 100644 --- a/llama_stack/cli/stack/run.py +++ b/llama_stack/cli/stack/run.py @@ -6,11 +6,18 @@ import argparse import os +import ssl import subprocess from pathlib import Path +import uvicorn +import yaml + from llama_stack.cli.stack.utils import ImageType from llama_stack.cli.subcommand import Subcommand +from llama_stack.core.datatypes import LoggingConfig, StackRunConfig +from llama_stack.core.stack import cast_image_name_to_string, replace_env_vars, validate_env_pair +from llama_stack.core.utils.config_resolution import Mode, resolve_config_or_distro from llama_stack.log import get_logger REPO_ROOT = Path(__file__).parent.parent.parent.parent @@ -146,23 +153,7 @@ class StackRun(Subcommand): # using the current environment packages. if not image_type and not image_name: logger.info("No image type or image name provided. Assuming environment packages.") - from llama_stack.core.server.server import main as server_main - - # Build the server args from the current args passed to the CLI - server_args = argparse.Namespace() - for arg in vars(args): - # If this is a function, avoid passing it - # "args" contains: - # func=> - if callable(getattr(args, arg)): - continue - if arg == "config": - server_args.config = str(config_file) - else: - setattr(server_args, arg, getattr(args, arg)) - - # Run the server - server_main(server_args) + self._uvicorn_run(config_file, args) else: run_args = formulate_run_args(image_type, image_name) @@ -184,6 +175,76 @@ class StackRun(Subcommand): run_command(run_args) + def _uvicorn_run(self, config_file: Path | None, args: argparse.Namespace) -> None: + if not config_file: + self.parser.error("Config file is required") + + # Set environment variables if provided + if args.env: + for env_pair in args.env: + try: + key, value = validate_env_pair(env_pair) + logger.info(f"Setting environment variable {key} => {value}") + os.environ[key] = value + except ValueError as e: + logger.error(f"Error: {str(e)}") + self.parser.error(f"Invalid environment variable format: {env_pair}") + + config_file = resolve_config_or_distro(str(config_file), Mode.RUN) + with open(config_file) as fp: + config_contents = yaml.safe_load(fp) + if isinstance(config_contents, dict) and (cfg := config_contents.get("logging_config")): + logger_config = LoggingConfig(**cfg) + else: + logger_config = None + config = StackRunConfig(**cast_image_name_to_string(replace_env_vars(config_contents))) + + port = args.port or config.server.port + host = config.server.host or "::" + + # Set the config file in environment so create_app can find it + os.environ["LLAMA_STACK_CONFIG"] = str(config_file) + + uvicorn_config = { + "factory": True, + "host": host, + "port": port, + "lifespan": "on", + "log_level": logger.getEffectiveLevel(), + "log_config": logger_config, + } + + keyfile = config.server.tls_keyfile + certfile = config.server.tls_certfile + if keyfile and certfile: + uvicorn_config["ssl_keyfile"] = config.server.tls_keyfile + uvicorn_config["ssl_certfile"] = config.server.tls_certfile + if config.server.tls_cafile: + uvicorn_config["ssl_ca_certs"] = config.server.tls_cafile + uvicorn_config["ssl_cert_reqs"] = ssl.CERT_REQUIRED + + logger.info( + f"HTTPS enabled with certificates:\n Key: {keyfile}\n Cert: {certfile}\n CA: {config.server.tls_cafile}" + ) + else: + logger.info(f"HTTPS enabled with certificates:\n Key: {keyfile}\n Cert: {certfile}") + + logger.info(f"Listening on {host}:{port}") + + # We need to catch KeyboardInterrupt because uvicorn's signal handling + # re-raises SIGINT signals using signal.raise_signal(), which Python + # converts to KeyboardInterrupt. Without this catch, we'd get a confusing + # stack trace when using Ctrl+C or kill -2 (SIGINT). + # SIGTERM (kill -15) works fine without this because Python doesn't + # have a default handler for it. + # + # Another approach would be to ignore SIGINT entirely - let uvicorn handle it through its own + # signal handling but this is quite intrusive and not worth the effort. + try: + uvicorn.run("llama_stack.core.server.server:create_app", **uvicorn_config) + except (KeyboardInterrupt, SystemExit): + logger.info("Received interrupt signal, shutting down gracefully...") + def _start_ui_development_server(self, stack_server_port: int): logger.info("Attempting to start UI development server...") # Check if npm is available diff --git a/llama_stack/core/build_container.sh b/llama_stack/core/build_container.sh index 8e47fc592..09878f535 100755 --- a/llama_stack/core/build_container.sh +++ b/llama_stack/core/build_container.sh @@ -324,14 +324,14 @@ fi RUN pip uninstall -y uv EOF -# If a run config is provided, we use the --config flag +# If a run config is provided, we use the llama stack CLI if [[ -n "$run_config" ]]; then add_to_container << EOF -ENTRYPOINT ["python", "-m", "llama_stack.core.server.server", "$RUN_CONFIG_PATH"] +ENTRYPOINT ["llama", "stack", "run", "$RUN_CONFIG_PATH"] EOF elif [[ "$distro_or_config" != *.yaml ]]; then add_to_container << EOF -ENTRYPOINT ["python", "-m", "llama_stack.core.server.server", "$distro_or_config"] +ENTRYPOINT ["llama", "stack", "run", "$distro_or_config"] EOF fi diff --git a/llama_stack/core/server/server.py b/llama_stack/core/server/server.py index 32be57880..6b38e1ac6 100644 --- a/llama_stack/core/server/server.py +++ b/llama_stack/core/server/server.py @@ -4,7 +4,6 @@ # This source code is licensed under the terms described in the LICENSE file in # the root directory of this source tree. -import argparse import asyncio import concurrent.futures import functools @@ -12,7 +11,6 @@ import inspect import json import logging # allow-direct-logging import os -import ssl import sys import traceback import warnings @@ -35,7 +33,6 @@ from pydantic import BaseModel, ValidationError from llama_stack.apis.common.errors import ConflictError, ResourceNotFoundError from llama_stack.apis.common.responses import PaginatedResponse -from llama_stack.cli.utils import add_config_distro_args, get_config_from_args from llama_stack.core.access_control.access_control import AccessDeniedError from llama_stack.core.datatypes import ( AuthenticationRequiredError, @@ -55,7 +52,6 @@ from llama_stack.core.stack import ( Stack, cast_image_name_to_string, replace_env_vars, - validate_env_pair, ) from llama_stack.core.utils.config import redact_sensitive_fields from llama_stack.core.utils.config_resolution import Mode, resolve_config_or_distro @@ -333,23 +329,18 @@ class ClientVersionMiddleware: return await self.app(scope, receive, send) -def create_app( - config_file: str | None = None, - env_vars: list[str] | None = None, -) -> StackApp: +def create_app() -> StackApp: """Create and configure the FastAPI application. - Args: - config_file: Path to config file. If None, uses LLAMA_STACK_CONFIG env var or default resolution. - env_vars: List of environment variables in KEY=value format. - disable_version_check: Whether to disable version checking. If None, uses LLAMA_STACK_DISABLE_VERSION_CHECK env var. + This factory function reads configuration from environment variables: + - LLAMA_STACK_CONFIG: Path to config file (required) Returns: Configured StackApp instance. """ - config_file = config_file or os.getenv("LLAMA_STACK_CONFIG") + config_file = os.getenv("LLAMA_STACK_CONFIG") if config_file is None: - raise ValueError("No config file provided and LLAMA_STACK_CONFIG env var is not set") + raise ValueError("LLAMA_STACK_CONFIG environment variable is required") config_file = resolve_config_or_distro(config_file, Mode.RUN) @@ -361,16 +352,6 @@ def create_app( logger_config = LoggingConfig(**cfg) logger = get_logger(name=__name__, category="core::server", config=logger_config) - if env_vars: - for env_pair in env_vars: - try: - key, value = validate_env_pair(env_pair) - logger.info(f"Setting environment variable {key} => {value}") - os.environ[key] = value - except ValueError as e: - logger.error(f"Error: {str(e)}") - raise ValueError(f"Invalid environment variable format: {env_pair}") from e - config = replace_env_vars(config_contents) config = StackRunConfig(**cast_image_name_to_string(config)) @@ -494,101 +475,6 @@ def create_app( return app -def main(args: argparse.Namespace | None = None): - """Start the LlamaStack server.""" - parser = argparse.ArgumentParser(description="Start the LlamaStack server.") - - add_config_distro_args(parser) - parser.add_argument( - "--port", - type=int, - default=int(os.getenv("LLAMA_STACK_PORT", 8321)), - help="Port to listen on", - ) - parser.add_argument( - "--env", - action="append", - help="Environment variables in KEY=value format. Can be specified multiple times.", - ) - - # Determine whether the server args are being passed by the "run" command, if this is the case - # the args will be passed as a Namespace object to the main function, otherwise they will be - # parsed from the command line - if args is None: - args = parser.parse_args() - - config_or_distro = get_config_from_args(args) - - try: - app = create_app( - config_file=config_or_distro, - env_vars=args.env, - ) - except Exception as e: - logger.error(f"Error creating app: {str(e)}") - sys.exit(1) - - config_file = resolve_config_or_distro(config_or_distro, Mode.RUN) - with open(config_file) as fp: - config_contents = yaml.safe_load(fp) - if isinstance(config_contents, dict) and (cfg := config_contents.get("logging_config")): - logger_config = LoggingConfig(**cfg) - else: - logger_config = None - config = StackRunConfig(**cast_image_name_to_string(replace_env_vars(config_contents))) - - import uvicorn - - # Configure SSL if certificates are provided - port = args.port or config.server.port - - ssl_config = None - keyfile = config.server.tls_keyfile - certfile = config.server.tls_certfile - - if keyfile and certfile: - ssl_config = { - "ssl_keyfile": keyfile, - "ssl_certfile": certfile, - } - if config.server.tls_cafile: - ssl_config["ssl_ca_certs"] = config.server.tls_cafile - ssl_config["ssl_cert_reqs"] = ssl.CERT_REQUIRED - logger.info( - f"HTTPS enabled with certificates:\n Key: {keyfile}\n Cert: {certfile}\n CA: {config.server.tls_cafile}" - ) - else: - logger.info(f"HTTPS enabled with certificates:\n Key: {keyfile}\n Cert: {certfile}") - - listen_host = config.server.host or ["::", "0.0.0.0"] - logger.info(f"Listening on {listen_host}:{port}") - - uvicorn_config = { - "app": app, - "host": listen_host, - "port": port, - "lifespan": "on", - "log_level": logger.getEffectiveLevel(), - "log_config": logger_config, - } - if ssl_config: - uvicorn_config.update(ssl_config) - - # We need to catch KeyboardInterrupt because uvicorn's signal handling - # re-raises SIGINT signals using signal.raise_signal(), which Python - # converts to KeyboardInterrupt. Without this catch, we'd get a confusing - # stack trace when using Ctrl+C or kill -2 (SIGINT). - # SIGTERM (kill -15) works fine without this because Python doesn't - # have a default handler for it. - # - # Another approach would be to ignore SIGINT entirely - let uvicorn handle it through its own - # signal handling but this is quite intrusive and not worth the effort. - try: - asyncio.run(uvicorn.Server(uvicorn.Config(**uvicorn_config)).serve()) - except (KeyboardInterrupt, SystemExit): - logger.info("Received interrupt signal, shutting down gracefully...") - - def _log_run_config(run_config: StackRunConfig): """Logs the run config with redacted fields and disabled providers removed.""" logger.info("Run configuration:") @@ -615,7 +501,3 @@ def remove_disabled_providers(obj): return [item for item in (remove_disabled_providers(i) for i in obj) if item is not None] else: return obj - - -if __name__ == "__main__": - main() diff --git a/llama_stack/core/start_stack.sh b/llama_stack/core/start_stack.sh index 4c6824b56..02b1cd408 100755 --- a/llama_stack/core/start_stack.sh +++ b/llama_stack/core/start_stack.sh @@ -116,7 +116,7 @@ if [[ "$env_type" == "venv" ]]; then yaml_config_arg="" fi - $PYTHON_BINARY -m llama_stack.core.server.server \ + llama stack run \ $yaml_config_arg \ --port "$port" \ $env_vars \