mirror of
https://github.com/BerriAI/litellm.git
synced 2025-04-25 18:54:30 +00:00
LiteLLM Minor Fixes & Improvements (10/10/2024) (#6158)
* refactor(vertex_ai_partner_models/anthropic): refactor anthropic to use partner model logic * fix(vertex_ai/): support passing custom api base to partner models Fixes https://github.com/BerriAI/litellm/issues/4317 * fix(proxy_server.py): Fix prometheus premium user check logic * docs(prometheus.md): update quick start docs * fix(custom_llm.py): support passing dynamic api key + api base * fix(realtime_api/main.py): Add request/response logging for realtime api endpoints Closes https://github.com/BerriAI/litellm/issues/6081 * feat(openai/realtime): add openai realtime api logging Closes https://github.com/BerriAI/litellm/issues/6081 * fix(realtime_streaming.py): fix linting errors * fix(realtime_streaming.py): fix linting errors * fix: fix linting errors * fix pattern match router * Add literalai in the sidebar observability category (#6163) * fix: add literalai in the sidebar * fix: typo * update (#6160) * Feat: Add Langtrace integration (#5341) * Feat: Add Langtrace integration * add langtrace service name * fix timestamps for traces * add tests * Discard Callback + use existing otel logger * cleanup * remove print statments * remove callback * add docs * docs * add logging docs * format logging * remove emoji and add litellm proxy example * format logging * format `logging.md` * add langtrace docs to logging.md * sync conflict * docs fix * (perf) move s3 logging to Batch logging + async [94% faster perf under 100 RPS on 1 litellm instance] (#6165) * fix move s3 to use customLogger * add basic s3 logging test * add s3 to custom logger compatible * use batch logger for s3 * s3 set flush interval and batch size * fix s3 logging * add notes on s3 logging * fix s3 logging * add basic s3 logging test * fix s3 type errors * add test for sync logging on s3 * fix: fix to debug log --------- Co-authored-by: Ishaan Jaff <ishaanjaffer0324@gmail.com> Co-authored-by: Willy Douhard <willy.douhard@gmail.com> Co-authored-by: yujonglee <yujonglee.dev@gmail.com> Co-authored-by: Ali Waleed <ali@scale3labs.com>
This commit is contained in:
Krish Dholakia
GitHub
parent
9db4ccca9f
commit
11f9df923a
28 changed files with 966 additions and 760 deletions
|
|
@ -117,10 +117,7 @@ from .llms.sagemaker.sagemaker import SagemakerLLM
|
|||
from .llms.text_completion_codestral import CodestralTextCompletion
|
||||
from .llms.together_ai.completion.handler import TogetherAITextCompletion
|
||||
from .llms.triton import TritonChatCompletion
|
||||
from .llms.vertex_ai_and_google_ai_studio import (
|
||||
vertex_ai_anthropic,
|
||||
vertex_ai_non_gemini,
|
||||
)
|
||||
from .llms.vertex_ai_and_google_ai_studio import vertex_ai_non_gemini
|
||||
from .llms.vertex_ai_and_google_ai_studio.gemini.vertex_and_google_ai_studio_gemini import (
|
||||
VertexLLM,
|
||||
)
|
||||
|
|
@ -747,6 +744,11 @@ def completion( # type: ignore
|
|||
proxy_server_request = kwargs.get("proxy_server_request", None)
|
||||
fallbacks = kwargs.get("fallbacks", None)
|
||||
headers = kwargs.get("headers", None) or extra_headers
|
||||
if headers is None:
|
||||
headers = {}
|
||||
|
||||
if extra_headers is not None:
|
||||
headers.update(extra_headers)
|
||||
num_retries = kwargs.get(
|
||||
"num_retries", None
|
||||
) ## alt. param for 'max_retries'. Use this to pass retries w/ instructor.
|
||||
|
|
@ -964,7 +966,6 @@ def completion( # type: ignore
|
|||
max_retries=max_retries,
|
||||
logprobs=logprobs,
|
||||
top_logprobs=top_logprobs,
|
||||
extra_headers=extra_headers,
|
||||
api_version=api_version,
|
||||
parallel_tool_calls=parallel_tool_calls,
|
||||
messages=messages,
|
||||
|
|
@ -1067,6 +1068,9 @@ def completion( # type: ignore
|
|||
|
||||
headers = headers or litellm.headers
|
||||
|
||||
if extra_headers is not None:
|
||||
optional_params["extra_headers"] = extra_headers
|
||||
|
||||
if (
|
||||
litellm.enable_preview_features
|
||||
and litellm.AzureOpenAIO1Config().is_o1_model(model=model)
|
||||
|
|
@ -1166,6 +1170,9 @@ def completion( # type: ignore
|
|||
|
||||
headers = headers or litellm.headers
|
||||
|
||||
if extra_headers is not None:
|
||||
optional_params["extra_headers"] = extra_headers
|
||||
|
||||
## LOAD CONFIG - if set
|
||||
config = litellm.AzureOpenAIConfig.get_config()
|
||||
for k, v in config.items():
|
||||
|
|
@ -1223,6 +1230,9 @@ def completion( # type: ignore
|
|||
|
||||
headers = headers or litellm.headers
|
||||
|
||||
if extra_headers is not None:
|
||||
optional_params["extra_headers"] = extra_headers
|
||||
|
||||
## LOAD CONFIG - if set
|
||||
config = litellm.AzureAIStudioConfig.get_config()
|
||||
for k, v in config.items():
|
||||
|
|
@ -1304,6 +1314,9 @@ def completion( # type: ignore
|
|||
|
||||
headers = headers or litellm.headers
|
||||
|
||||
if extra_headers is not None:
|
||||
optional_params["extra_headers"] = extra_headers
|
||||
|
||||
## LOAD CONFIG - if set
|
||||
config = litellm.OpenAITextCompletionConfig.get_config()
|
||||
for k, v in config.items():
|
||||
|
|
@ -1466,6 +1479,9 @@ def completion( # type: ignore
|
|||
|
||||
headers = headers or litellm.headers
|
||||
|
||||
if extra_headers is not None:
|
||||
optional_params["extra_headers"] = extra_headers
|
||||
|
||||
## LOAD CONFIG - if set
|
||||
config = litellm.OpenAIConfig.get_config()
|
||||
for k, v in config.items():
|
||||
|
|
@ -2228,31 +2244,12 @@ def completion( # type: ignore
|
|||
)
|
||||
|
||||
new_params = deepcopy(optional_params)
|
||||
if "claude-3" in model:
|
||||
model_response = vertex_ai_anthropic.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
model_response=model_response,
|
||||
print_verbose=print_verbose,
|
||||
optional_params=new_params,
|
||||
litellm_params=litellm_params,
|
||||
logger_fn=logger_fn,
|
||||
encoding=encoding,
|
||||
vertex_location=vertex_ai_location,
|
||||
vertex_project=vertex_ai_project,
|
||||
vertex_credentials=vertex_credentials,
|
||||
logging_obj=logging,
|
||||
acompletion=acompletion,
|
||||
headers=headers,
|
||||
custom_prompt_dict=custom_prompt_dict,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
)
|
||||
elif (
|
||||
if (
|
||||
model.startswith("meta/")
|
||||
or model.startswith("mistral")
|
||||
or model.startswith("codestral")
|
||||
or model.startswith("jamba")
|
||||
or model.startswith("claude")
|
||||
):
|
||||
model_response = vertex_partner_models_chat_completion.completion(
|
||||
model=model,
|
||||
|
|
@ -2263,6 +2260,7 @@ def completion( # type: ignore
|
|||
litellm_params=litellm_params, # type: ignore
|
||||
logger_fn=logger_fn,
|
||||
encoding=encoding,
|
||||
api_base=api_base,
|
||||
vertex_location=vertex_ai_location,
|
||||
vertex_project=vertex_ai_project,
|
||||
vertex_credentials=vertex_credentials,
|
||||
|
|
@ -4848,6 +4846,8 @@ def image_generation(
|
|||
model_response = custom_handler.aimage_generation( # type: ignore
|
||||
model=model,
|
||||
prompt=prompt,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
model_response=model_response,
|
||||
optional_params=optional_params,
|
||||
logging_obj=litellm_logging_obj,
|
||||
|
|
@ -4863,6 +4863,8 @@ def image_generation(
|
|||
model_response = custom_handler.image_generation(
|
||||
model=model,
|
||||
prompt=prompt,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
model_response=model_response,
|
||||
optional_params=optional_params,
|
||||
logging_obj=litellm_logging_obj,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue