mirror of
https://github.com/BerriAI/litellm.git
synced 2025-04-24 18:24:20 +00:00
(docs) controlling num_workers using env variables
This commit is contained in:
parent
893627cab4
commit
a84c86562c
1 changed files with 43 additions and 18 deletions
|
@ -1,31 +1,46 @@
|
|||
# CLI Arguments
|
||||
Cli arguments, --host, --port, --num_workers
|
||||
|
||||
#### --host
|
||||
## --host
|
||||
- **Default:** `'0.0.0.0'`
|
||||
- The host for the server to listen on.
|
||||
- **Usage:**
|
||||
```shell
|
||||
litellm --host 127.0.0.1
|
||||
```
|
||||
- **Usage - set Environment Variable:** `HOST`
|
||||
```shell
|
||||
export HOST=127.0.0.1
|
||||
litellm
|
||||
```
|
||||
|
||||
#### --port
|
||||
## --port
|
||||
- **Default:** `8000`
|
||||
- The port to bind the server to.
|
||||
- **Usage:**
|
||||
```shell
|
||||
litellm --port 8080
|
||||
```
|
||||
- **Usage - set Environment Variable:** `PORT`
|
||||
```shell
|
||||
export PORT=8080
|
||||
litellm
|
||||
```
|
||||
|
||||
#### --num_workers
|
||||
## --num_workers
|
||||
- **Default:** `1`
|
||||
- The number of uvicorn workers to spin up.
|
||||
- **Usage:**
|
||||
```shell
|
||||
litellm --num_workers 4
|
||||
```
|
||||
- **Usage - set Environment Variable:** `NUM_WORKERS`
|
||||
```shell
|
||||
export NUM_WORKERS=4
|
||||
litellm
|
||||
```
|
||||
|
||||
#### --api_base
|
||||
## --api_base
|
||||
- **Default:** `None`
|
||||
- The API base for the model litellm should call.
|
||||
- **Usage:**
|
||||
|
@ -33,7 +48,7 @@ Cli arguments, --host, --port, --num_workers
|
|||
litellm --model huggingface/tinyllama --api_base https://k58ory32yinf1ly0.us-east-1.aws.endpoints.huggingface.cloud
|
||||
```
|
||||
|
||||
#### --api_version
|
||||
## --api_version
|
||||
- **Default:** `None`
|
||||
- For Azure services, specify the API version.
|
||||
- **Usage:**
|
||||
|
@ -41,7 +56,7 @@ Cli arguments, --host, --port, --num_workers
|
|||
litellm --model azure/gpt-deployment --api_version 2023-08-01 --api_base https://<your api base>"
|
||||
```
|
||||
|
||||
#### --model or -m
|
||||
## --model or -m
|
||||
- **Default:** `None`
|
||||
- The model name to pass to Litellm.
|
||||
- **Usage:**
|
||||
|
@ -49,7 +64,7 @@ Cli arguments, --host, --port, --num_workers
|
|||
litellm --model gpt-3.5-turbo
|
||||
```
|
||||
|
||||
#### --test
|
||||
## --test
|
||||
- **Type:** `bool` (Flag)
|
||||
- Proxy chat completions URL to make a test request.
|
||||
- **Usage:**
|
||||
|
@ -57,7 +72,7 @@ Cli arguments, --host, --port, --num_workers
|
|||
litellm --test
|
||||
```
|
||||
|
||||
#### --health
|
||||
## --health
|
||||
- **Type:** `bool` (Flag)
|
||||
- Runs a health check on all models in config.yaml
|
||||
- **Usage:**
|
||||
|
@ -65,7 +80,7 @@ Cli arguments, --host, --port, --num_workers
|
|||
litellm --health
|
||||
```
|
||||
|
||||
#### --alias
|
||||
## --alias
|
||||
- **Default:** `None`
|
||||
- An alias for the model, for user-friendly reference.
|
||||
- **Usage:**
|
||||
|
@ -73,7 +88,7 @@ Cli arguments, --host, --port, --num_workers
|
|||
litellm --alias my-gpt-model
|
||||
```
|
||||
|
||||
#### --debug
|
||||
## --debug
|
||||
- **Default:** `False`
|
||||
- **Type:** `bool` (Flag)
|
||||
- Enable debugging mode for the input.
|
||||
|
@ -81,15 +96,25 @@ Cli arguments, --host, --port, --num_workers
|
|||
```shell
|
||||
litellm --debug
|
||||
```
|
||||
- **Usage - set Environment Variable:** `DEBUG`
|
||||
```shell
|
||||
export DEBUG=True
|
||||
litellm
|
||||
```
|
||||
|
||||
#### --detailed_debug
|
||||
## --detailed_debug
|
||||
- **Default:** `False`
|
||||
- **Type:** `bool` (Flag)
|
||||
- Enable debugging mode for the input.
|
||||
- **Usage:**
|
||||
```shell
|
||||
litellm --detailed_debug
|
||||
``
|
||||
```
|
||||
- **Usage - set Environment Variable:** `DETAILED_DEBUG`
|
||||
```shell
|
||||
export DETAILED_DEBUG=True
|
||||
litellm
|
||||
```
|
||||
|
||||
#### --temperature
|
||||
- **Default:** `None`
|
||||
|
@ -100,7 +125,7 @@ Cli arguments, --host, --port, --num_workers
|
|||
litellm --temperature 0.7
|
||||
```
|
||||
|
||||
#### --max_tokens
|
||||
## --max_tokens
|
||||
- **Default:** `None`
|
||||
- **Type:** `int`
|
||||
- Set the maximum number of tokens for the model output.
|
||||
|
@ -109,7 +134,7 @@ Cli arguments, --host, --port, --num_workers
|
|||
litellm --max_tokens 50
|
||||
```
|
||||
|
||||
#### --request_timeout
|
||||
## --request_timeout
|
||||
- **Default:** `600`
|
||||
- **Type:** `int`
|
||||
- Set the timeout in seconds for completion calls.
|
||||
|
@ -118,7 +143,7 @@ Cli arguments, --host, --port, --num_workers
|
|||
litellm --request_timeout 300
|
||||
```
|
||||
|
||||
#### --drop_params
|
||||
## --drop_params
|
||||
- **Type:** `bool` (Flag)
|
||||
- Drop any unmapped params.
|
||||
- **Usage:**
|
||||
|
@ -126,7 +151,7 @@ Cli arguments, --host, --port, --num_workers
|
|||
litellm --drop_params
|
||||
```
|
||||
|
||||
#### --add_function_to_prompt
|
||||
## --add_function_to_prompt
|
||||
- **Type:** `bool` (Flag)
|
||||
- If a function passed but unsupported, pass it as a part of the prompt.
|
||||
- **Usage:**
|
||||
|
@ -134,14 +159,14 @@ Cli arguments, --host, --port, --num_workers
|
|||
litellm --add_function_to_prompt
|
||||
```
|
||||
|
||||
#### --config
|
||||
## --config
|
||||
- Configure Litellm by providing a configuration file path.
|
||||
- **Usage:**
|
||||
```shell
|
||||
litellm --config path/to/config.yaml
|
||||
```
|
||||
|
||||
#### --telemetry
|
||||
## --telemetry
|
||||
- **Default:** `True`
|
||||
- **Type:** `bool`
|
||||
- Help track usage of this feature.
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue