diff --git a/llama_stack/cli/stack/run.py b/llama_stack/cli/stack/run.py index f84def184..e7d6df292 100644 --- a/llama_stack/cli/stack/run.py +++ b/llama_stack/cli/stack/run.py @@ -55,6 +55,16 @@ class StackRun(Subcommand): default=[], metavar="KEY=VALUE", ) + self.parser.add_argument( + "--tls-keyfile", + type=str, + help="Path to TLS key file for HTTPS", + ) + self.parser.add_argument( + "--tls-certfile", + type=str, + help="Path to TLS certificate file for HTTPS", + ) def _run_stack_run_cmd(self, args: argparse.Namespace) -> None: import importlib.resources @@ -178,4 +188,7 @@ class StackRun(Subcommand): return run_args.extend(["--env", f"{key}={value}"]) + if args.tls_keyfile and args.tls_certfile: + run_args.extend(["--tls-keyfile", args.tls_keyfile, "--tls-certfile", args.tls_certfile]) + run_with_pty(run_args) diff --git a/llama_stack/distribution/datatypes.py b/llama_stack/distribution/datatypes.py index 8b579b636..97706f22a 100644 --- a/llama_stack/distribution/datatypes.py +++ b/llama_stack/distribution/datatypes.py @@ -117,6 +117,23 @@ class Provider(BaseModel): config: Dict[str, Any] +class ServerConfig(BaseModel): + port: int = Field( + default=8321, + description="Port to listen on", + ge=1024, + le=65535, + ) + tls_certfile: Optional[str] = Field( + default=None, + description="Path to TLS certificate file for HTTPS", + ) + tls_keyfile: Optional[str] = Field( + default=None, + description="Path to TLS key file for HTTPS", + ) + + class StackRunConfig(BaseModel): version: str = LLAMA_STACK_RUN_CONFIG_VERSION @@ -159,6 +176,11 @@ a default SQLite store will be used.""", eval_tasks: List[EvalTaskInput] = Field(default_factory=list) tool_groups: List[ToolGroupInput] = Field(default_factory=list) + server: ServerConfig = Field( + default_factory=ServerConfig, + description="Configuration for the HTTP(S) server", + ) + class BuildConfig(BaseModel): version: str = LLAMA_STACK_BUILD_CONFIG_VERSION diff --git a/llama_stack/distribution/server/server.py b/llama_stack/distribution/server/server.py index fcd0e3cad..d2c32de11 100644 --- a/llama_stack/distribution/server/server.py +++ b/llama_stack/distribution/server/server.py @@ -282,8 +282,19 @@ def main(): action="append", help="Environment variables in KEY=value format. Can be specified multiple times.", ) + parser.add_argument( + "--tls-keyfile", + help="Path to TLS key file for HTTPS", + required="--tls-certfile" in sys.argv, + ) + parser.add_argument( + "--tls-certfile", + help="Path to TLS certificate file for HTTPS", + required="--tls-keyfile" in sys.argv, + ) args = parser.parse_args() + if args.env: for env_pair in args.env: try: @@ -381,11 +392,36 @@ def main(): import uvicorn - # FYI this does not do hot-reloads + # Configure SSL if certificates are provided + port = args.port or config.server.port + + ssl_config = None + if args.tls_keyfile: + keyfile = args.tls_keyfile + certfile = args.tls_certfile + else: + keyfile = config.server.tls_keyfile + certfile = config.server.tls_certfile + + if keyfile and certfile: + ssl_config = { + "ssl_keyfile": keyfile, + "ssl_certfile": certfile, + } + print(f"HTTPS enabled with certificates:\n Key: {keyfile}\n Cert: {certfile}") listen_host = ["::", "0.0.0.0"] if not args.disable_ipv6 else "0.0.0.0" - print(f"Listening on {listen_host}:{args.port}") - uvicorn.run(app, host=listen_host, port=args.port) + print(f"Listening on {listen_host}:{port}") + + uvicorn_config = { + "app": app, + "host": listen_host, + "port": port, + } + if ssl_config: + uvicorn_config.update(ssl_config) + + uvicorn.run(**uvicorn_config) def extract_path_params(route: str) -> List[str]: diff --git a/llama_stack/distribution/start_conda_env.sh b/llama_stack/distribution/start_conda_env.sh index c37f30ef0..fe830059f 100755 --- a/llama_stack/distribution/start_conda_env.sh +++ b/llama_stack/distribution/start_conda_env.sh @@ -34,6 +34,7 @@ shift # Process environment variables from --env arguments env_vars="" +other_args="" while [[ $# -gt 0 ]]; do case "$1" in --env) @@ -48,6 +49,7 @@ while [[ $# -gt 0 ]]; do fi ;; *) + other_args="$other_args $1" shift ;; esac @@ -61,4 +63,5 @@ $CONDA_PREFIX/bin/python \ -m llama_stack.distribution.server.server \ --yaml-config "$yaml_config" \ --port "$port" \ - $env_vars + $env_vars \ + $other_args diff --git a/llama_stack/distribution/start_container.sh b/llama_stack/distribution/start_container.sh index 2c5d65d09..a5f543fb4 100755 --- a/llama_stack/distribution/start_container.sh +++ b/llama_stack/distribution/start_container.sh @@ -40,8 +40,12 @@ shift port="$1" shift +# Initialize other_args +other_args="" + # Process environment variables from --env arguments env_vars="" + while [[ $# -gt 0 ]]; do case "$1" in --env) @@ -55,6 +59,7 @@ while [[ $# -gt 0 ]]; do fi ;; *) + other_args="$other_args $1" shift ;; esac @@ -93,5 +98,8 @@ $CONTAINER_BINARY run $CONTAINER_OPTS -it \ -v "$yaml_config:/app/config.yaml" \ $mounts \ --env LLAMA_STACK_PORT=$port \ - --entrypoint='["python", "-m", "llama_stack.distribution.server.server", "--yaml-config", "/app/config.yaml"]' \ - $container_image:$version_tag + --entrypoint python \ + $container_image:$version_tag \ + -m llama_stack.distribution.server.server \ + --yaml-config /app/config.yaml \ + $other_args