diff --git a/docs/my-website/docs/providers/custom.md b/docs/my-website/docs/providers/custom.md new file mode 100644 index 000000000..4f2af36a8 --- /dev/null +++ b/docs/my-website/docs/providers/custom.md @@ -0,0 +1,53 @@ +# Custom LLM API-Endpoints +LiteLLM supports Custom deploy api endpoints + +LiteLLM Expects the following input and output for custom LLM API endpoints +#### Input +Inputs to your custom LLM api bases should follow this format: + +```python +resp = requests.post( + your-api_base, + json={ + 'model': 'meta-llama/Llama-2-13b-hf', # model name + 'params': { + 'prompt': ["The capital of France is P"], + 'max_tokens': 32, + 'temperature': 0.7, + 'top_p': 1.0, + 'top_k': 40, + } + } +) +``` + +#### Output +Outputs from your custom LLM api bases should follow this format: +""" +{ + 'data': [ + { + 'prompt': 'The capital of France is P', + 'output': [ + 'The capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France' + ], + 'params': { + 'temperature': 0.7, + 'top_k': 40, + 'top_p': 1 + } + } + ], + 'message': 'ok' +} +""" + +### Model Details + +For calls to your custom API base ensure: +* Set `api_base="your-api-base"` +* Add `custom/` as a prefix to the `model` param. If your API expects `meta-llama/Llama-2-13b-hf` set `model=custom/meta-llama/Llama-2-13b-hf` + +| Model Name | Function Call | +|------------------|--------------------------------------------| +| meta-llama/Llama-2-13b-hf | `response = completion(model="custom/meta-llama/Llama-2-13b-hf", messages=messages, api_base="https://your-custom-inference-endpoint")` | diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index f274dbf09..61901a009 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -38,7 +38,7 @@ const sidebars = { description: 'Learn how to deploy + call models from different providers on LiteLLM', slug: '/providers', }, - items: ["providers/huggingface", "providers/openai", "providers/azure", "providers/vertex", "providers/anthropic", "providers/ai21", "providers/replicate", "providers/cohere", "providers/togetherai", "providers/aws_sagemaker", "providers/aleph_alpha", "providers/baseten", "providers/openrouter", "providers/ollama"] + items: ["providers/huggingface", "providers/openai", "providers/azure", "providers/vertex", "providers/anthropic", "providers/ai21", "providers/replicate", "providers/cohere", "providers/togetherai", "providers/aws_sagemaker", "providers/aleph_alpha", "providers/baseten", "providers/openrouter", "providers/ollama", "providers/custom"] }, "token_usage", "exception_mapping",