mirror of
https://github.com/BerriAI/litellm.git
synced 2025-04-26 19:24:27 +00:00
e95820367f
* fix(hosted_vllm/transformation.py): return fake api key, if none give. Prevents httpx error Fixes https://github.com/BerriAI/litellm/issues/7291 * test: fix test * fix(main.py): add hosted_vllm/ support for embeddings endpoint Closes https://github.com/BerriAI/litellm/issues/7290 * docs(vllm.md): add docs on vllm embeddings usage * fix(__init__.py): fix sambanova model test * fix(base_llm_unit_tests.py): skip pydantic obj test if model takes >5s to respond
45 lines
1.6 KiB
Python
45 lines
1.6 KiB
Python
"""
|
|
Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions`
|
|
"""
|
|
|
|
import types
|
|
from typing import List, Optional, Tuple, Union
|
|
|
|
from pydantic import BaseModel
|
|
|
|
import litellm
|
|
from litellm.secret_managers.main import get_secret_str
|
|
from litellm.types.llms.openai import AllMessageValues, ChatCompletionAssistantMessage
|
|
|
|
from ....utils import _remove_additional_properties, _remove_strict_from_schema
|
|
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
|
|
|
|
|
|
class HostedVLLMChatConfig(OpenAIGPTConfig):
|
|
def map_openai_params(
|
|
self,
|
|
non_default_params: dict,
|
|
optional_params: dict,
|
|
model: str,
|
|
drop_params: bool,
|
|
) -> dict:
|
|
_tools = non_default_params.pop("tools", None)
|
|
if _tools is not None:
|
|
# remove 'additionalProperties' from tools
|
|
_tools = _remove_additional_properties(_tools)
|
|
# remove 'strict' from tools
|
|
_tools = _remove_strict_from_schema(_tools)
|
|
if _tools is not None:
|
|
non_default_params["tools"] = _tools
|
|
return super().map_openai_params(
|
|
non_default_params, optional_params, model, drop_params
|
|
)
|
|
|
|
def _get_openai_compatible_provider_info(
|
|
self, api_base: Optional[str], api_key: Optional[str]
|
|
) -> Tuple[Optional[str], Optional[str]]:
|
|
api_base = api_base or get_secret_str("HOSTED_VLLM_API_BASE") # type: ignore
|
|
dynamic_api_key = (
|
|
api_key or get_secret_str("HOSTED_VLLM_API_KEY") or "fake-api-key"
|
|
) # vllm does not require an api key
|
|
return api_base, dynamic_api_key
|