forked from phoenix/litellm-mirror
Merge branch 'main' into litellm_block_unblock_user_api
This commit is contained in:
Krish Dholakia
GitHub
commit
aaf086c0a8
17 changed files with 771 additions and 438 deletions
|
|
@ -121,7 +121,7 @@ response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content
|
|||
|
||||
# OpenAI Proxy - ([Docs](https://docs.litellm.ai/docs/simple_proxy))
|
||||
|
||||
Track spend across multiple projects/people
|
||||
Set Budgets & Rate limits across multiple projects
|
||||
|
||||
The proxy provides:
|
||||
1. [Hooks for auth](https://docs.litellm.ai/docs/proxy/virtual_keys#custom-auth)
|
||||
|
|
@ -163,7 +163,7 @@ print(response)
|
|||
UI on `/ui` on your proxy server
|
||||
|
||||
|
||||
Track Spend, Set budgets and create virtual keys for the proxy
|
||||
Set budgets and rate limits across multiple projects
|
||||
`POST /key/generate`
|
||||
|
||||
### Request
|
||||
|
|
|
|||
|
|
@ -165,6 +165,7 @@ s3_callback_params: Optional[Dict] = None
|
|||
generic_logger_headers: Optional[Dict] = None
|
||||
default_key_generate_params: Optional[Dict] = None
|
||||
upperbound_key_generate_params: Optional[Dict] = None
|
||||
default_user_params: Optional[Dict] = None
|
||||
default_team_settings: Optional[List] = None
|
||||
max_user_budget: Optional[float] = None
|
||||
#### RELIABILITY ####
|
||||
|
|
|
|||
|
|
@ -235,6 +235,9 @@ class LangFuseLogger:
|
|||
supports_tags = Version(langfuse.version.__version__) >= Version("2.6.3")
|
||||
supports_prompt = Version(langfuse.version.__version__) >= Version("2.7.3")
|
||||
supports_costs = Version(langfuse.version.__version__) >= Version("2.7.3")
|
||||
supports_completion_start_time = Version(
|
||||
langfuse.version.__version__
|
||||
) >= Version("2.7.3")
|
||||
|
||||
print_verbose(f"Langfuse Layer Logging - logging to langfuse v2 ")
|
||||
|
||||
|
|
@ -308,6 +311,11 @@ class LangFuseLogger:
|
|||
if output is not None and isinstance(output, str) and level == "ERROR":
|
||||
generation_params["statusMessage"] = output
|
||||
|
||||
if supports_completion_start_time:
|
||||
generation_params["completion_start_time"] = kwargs.get(
|
||||
"completion_start_time", None
|
||||
)
|
||||
|
||||
trace.generation(**generation_params)
|
||||
except Exception as e:
|
||||
print(f"Langfuse Layer Error - {traceback.format_exc()}")
|
||||
|
|
|
|||
|
|
@ -575,12 +575,20 @@ class Huggingface(BaseLLM):
|
|||
response = await client.post(url=api_base, json=data, headers=headers)
|
||||
response_json = response.json()
|
||||
if response.status_code != 200:
|
||||
raise HuggingfaceError(
|
||||
status_code=response.status_code,
|
||||
message=response.text,
|
||||
request=response.request,
|
||||
response=response,
|
||||
)
|
||||
if "error" in response_json:
|
||||
raise HuggingfaceError(
|
||||
status_code=response.status_code,
|
||||
message=response_json["error"],
|
||||
request=response.request,
|
||||
response=response,
|
||||
)
|
||||
else:
|
||||
raise HuggingfaceError(
|
||||
status_code=response.status_code,
|
||||
message=response.text,
|
||||
request=response.request,
|
||||
response=response,
|
||||
)
|
||||
|
||||
## RESPONSE OBJECT
|
||||
return self.convert_to_model_response_object(
|
||||
|
|
@ -595,6 +603,8 @@ class Huggingface(BaseLLM):
|
|||
except Exception as e:
|
||||
if isinstance(e, httpx.TimeoutException):
|
||||
raise HuggingfaceError(status_code=500, message="Request Timeout Error")
|
||||
elif isinstance(e, HuggingfaceError):
|
||||
raise e
|
||||
elif response is not None and hasattr(response, "text"):
|
||||
raise HuggingfaceError(
|
||||
status_code=500,
|
||||
|
|
|
|||
|
|
@ -730,6 +730,8 @@ async def user_api_key_auth(
|
|||
"/user",
|
||||
"/model/info",
|
||||
"/v2/model/info",
|
||||
"/models",
|
||||
"/v1/models",
|
||||
]
|
||||
# check if the current route startswith any of the allowed routes
|
||||
if (
|
||||
|
|
@ -1758,6 +1760,7 @@ async def generate_key_helper_fn(
|
|||
allowed_cache_controls: Optional[list] = [],
|
||||
permissions: Optional[dict] = {},
|
||||
model_max_budget: Optional[dict] = {},
|
||||
table_name: Optional[Literal["key", "user"]] = None,
|
||||
):
|
||||
global prisma_client, custom_db_client, user_api_key_cache
|
||||
|
||||
|
|
@ -1884,8 +1887,10 @@ async def generate_key_helper_fn(
|
|||
table_name="user",
|
||||
update_key_values=update_key_values,
|
||||
)
|
||||
if user_id == litellm_proxy_budget_name:
|
||||
# do not create a key for litellm_proxy_budget_name
|
||||
if user_id == litellm_proxy_budget_name or (
|
||||
table_name is not None and table_name == "user"
|
||||
):
|
||||
# do not create a key for litellm_proxy_budget_name or if table name is set to just 'user'
|
||||
# we only need to ensure this exists in the user table
|
||||
# the LiteLLM_VerificationToken table will increase in size if we don't do this check
|
||||
return key_data
|
||||
|
|
@ -2440,7 +2445,7 @@ async def completion(
|
|||
)
|
||||
traceback.print_exc()
|
||||
error_traceback = traceback.format_exc()
|
||||
error_msg = f"{str(e)}\n\n{error_traceback}"
|
||||
error_msg = f"{str(e)}"
|
||||
raise ProxyException(
|
||||
message=getattr(e, "message", error_msg),
|
||||
type=getattr(e, "type", "None"),
|
||||
|
|
@ -5548,27 +5553,50 @@ async def auth_callback(request: Request):
|
|||
user_id_models: List = []
|
||||
|
||||
# User might not be already created on first generation of key
|
||||
# But if it is, we want its models preferences
|
||||
# But if it is, we want their models preferences
|
||||
default_ui_key_values = {
|
||||
"duration": "1hr",
|
||||
"key_max_budget": 0.01,
|
||||
"aliases": {},
|
||||
"config": {},
|
||||
"spend": 0,
|
||||
"team_id": "litellm-dashboard",
|
||||
}
|
||||
user_defined_values = {
|
||||
"models": user_id_models,
|
||||
"user_id": user_id,
|
||||
"user_email": user_email,
|
||||
}
|
||||
try:
|
||||
if prisma_client is not None:
|
||||
user_info = await prisma_client.get_data(user_id=user_id, table_name="user")
|
||||
verbose_proxy_logger.debug(
|
||||
f"user_info: {user_info}; litellm.default_user_params: {litellm.default_user_params}"
|
||||
)
|
||||
if user_info is not None:
|
||||
user_id_models = getattr(user_info, "models", [])
|
||||
user_defined_values = {
|
||||
"models": getattr(user_info, "models", []),
|
||||
"user_id": getattr(user_info, "user_id", user_id),
|
||||
"user_email": getattr(user_info, "user_id", user_email),
|
||||
}
|
||||
elif litellm.default_user_params is not None and isinstance(
|
||||
litellm.default_user_params, dict
|
||||
):
|
||||
user_defined_values = {
|
||||
"models": litellm.default_user_params.get("models", user_id_models),
|
||||
"user_id": litellm.default_user_params.get("user_id", user_id),
|
||||
"user_email": litellm.default_user_params.get(
|
||||
"user_email", user_email
|
||||
),
|
||||
}
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
verbose_proxy_logger.info(
|
||||
f"user_defined_values for creating ui key: {user_defined_values}"
|
||||
)
|
||||
response = await generate_key_helper_fn(
|
||||
**{
|
||||
"duration": "1hr",
|
||||
"key_max_budget": 0.01,
|
||||
"models": user_id_models,
|
||||
"aliases": {},
|
||||
"config": {},
|
||||
"spend": 0,
|
||||
"user_id": user_id,
|
||||
"team_id": "litellm-dashboard",
|
||||
"user_email": user_email,
|
||||
} # type: ignore
|
||||
**default_ui_key_values, **user_defined_values # type: ignore
|
||||
)
|
||||
key = response["token"] # type: ignore
|
||||
user_id = response["user_id"] # type: ignore
|
||||
|
|
|
|||
|
|
@ -554,6 +554,11 @@ class PrismaClient:
|
|||
f"PrismaClient: find_unique for token: {hashed_token}"
|
||||
)
|
||||
if query_type == "find_unique":
|
||||
if token is None:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"error": f"No token passed in. Token={token}"},
|
||||
)
|
||||
response = await self.db.litellm_verificationtoken.find_unique(
|
||||
where={"token": hashed_token}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -830,8 +830,8 @@ class Router:
|
|||
verbose_router_logger.info(
|
||||
f"litellm.atext_completion(model={model})\033[31m Exception {str(e)}\033[0m"
|
||||
)
|
||||
if model_name is not None:
|
||||
self.fail_calls[model_name] += 1
|
||||
if model is not None:
|
||||
self.fail_calls[model] += 1
|
||||
raise e
|
||||
|
||||
def embedding(
|
||||
|
|
|
|||
|
|
@ -206,6 +206,7 @@ def test_async_custom_handler_stream():
|
|||
# test_async_custom_handler_stream()
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="Flaky test")
|
||||
def test_azure_completion_stream():
|
||||
# [PROD Test] - Do not DELETE
|
||||
# test if completion() + sync custom logger get the same complete stream response
|
||||
|
|
|
|||
|
|
@ -1655,6 +1655,202 @@ def test_openai_streaming_and_function_calling():
|
|||
raise e
|
||||
|
||||
|
||||
# test_azure_streaming_and_function_calling()
|
||||
|
||||
|
||||
def test_success_callback_streaming():
|
||||
def success_callback(kwargs, completion_response, start_time, end_time):
|
||||
print(
|
||||
{
|
||||
"success": True,
|
||||
"input": kwargs,
|
||||
"output": completion_response,
|
||||
"start_time": start_time,
|
||||
"end_time": end_time,
|
||||
}
|
||||
)
|
||||
|
||||
litellm.success_callback = [success_callback]
|
||||
|
||||
messages = [{"role": "user", "content": "hello"}]
|
||||
print("TESTING LITELLM COMPLETION CALL")
|
||||
response = litellm.completion(
|
||||
model="j2-light",
|
||||
messages=messages,
|
||||
stream=True,
|
||||
max_tokens=5,
|
||||
)
|
||||
print(response)
|
||||
|
||||
for chunk in response:
|
||||
print(chunk["choices"][0])
|
||||
|
||||
|
||||
# test_success_callback_streaming()
|
||||
|
||||
#### STREAMING + FUNCTION CALLING ###
|
||||
from pydantic import BaseModel
|
||||
from typing import List, Optional
|
||||
|
||||
|
||||
class Function(BaseModel):
|
||||
name: str
|
||||
arguments: str
|
||||
|
||||
|
||||
class ToolCalls(BaseModel):
|
||||
index: int
|
||||
id: str
|
||||
type: str
|
||||
function: Function
|
||||
|
||||
|
||||
class Delta(BaseModel):
|
||||
role: str
|
||||
content: Optional[str]
|
||||
tool_calls: List[ToolCalls]
|
||||
|
||||
|
||||
class Choices(BaseModel):
|
||||
index: int
|
||||
delta: Delta
|
||||
logprobs: Optional[str]
|
||||
finish_reason: Optional[str]
|
||||
|
||||
|
||||
class Chunk(BaseModel):
|
||||
id: str
|
||||
object: str
|
||||
created: int
|
||||
model: str
|
||||
system_fingerprint: str
|
||||
choices: List[Choices]
|
||||
|
||||
|
||||
def validate_first_streaming_function_calling_chunk(chunk: ModelResponse):
|
||||
chunk_instance = Chunk(**chunk.model_dump())
|
||||
|
||||
|
||||
### Chunk 1
|
||||
|
||||
|
||||
# {
|
||||
# "id": "chatcmpl-8vdVjtzxc0JqGjq93NxC79dMp6Qcs",
|
||||
# "object": "chat.completion.chunk",
|
||||
# "created": 1708747267,
|
||||
# "model": "gpt-3.5-turbo-0125",
|
||||
# "system_fingerprint": "fp_86156a94a0",
|
||||
# "choices": [
|
||||
# {
|
||||
# "index": 0,
|
||||
# "delta": {
|
||||
# "role": "assistant",
|
||||
# "content": null,
|
||||
# "tool_calls": [
|
||||
# {
|
||||
# "index": 0,
|
||||
# "id": "call_oN10vaaC9iA8GLFRIFwjCsN7",
|
||||
# "type": "function",
|
||||
# "function": {
|
||||
# "name": "get_current_weather",
|
||||
# "arguments": ""
|
||||
# }
|
||||
# }
|
||||
# ]
|
||||
# },
|
||||
# "logprobs": null,
|
||||
# "finish_reason": null
|
||||
# }
|
||||
# ]
|
||||
# }
|
||||
class Function2(BaseModel):
|
||||
arguments: str
|
||||
|
||||
|
||||
class ToolCalls2(BaseModel):
|
||||
index: int
|
||||
function: Optional[Function2]
|
||||
|
||||
|
||||
class Delta2(BaseModel):
|
||||
tool_calls: List[ToolCalls2]
|
||||
|
||||
|
||||
class Choices2(BaseModel):
|
||||
index: int
|
||||
delta: Delta2
|
||||
logprobs: Optional[str]
|
||||
finish_reason: Optional[str]
|
||||
|
||||
|
||||
class Chunk2(BaseModel):
|
||||
id: str
|
||||
object: str
|
||||
created: int
|
||||
model: str
|
||||
system_fingerprint: str
|
||||
choices: List[Choices2]
|
||||
|
||||
|
||||
## Chunk 2
|
||||
|
||||
# {
|
||||
# "id": "chatcmpl-8vdVjtzxc0JqGjq93NxC79dMp6Qcs",
|
||||
# "object": "chat.completion.chunk",
|
||||
# "created": 1708747267,
|
||||
# "model": "gpt-3.5-turbo-0125",
|
||||
# "system_fingerprint": "fp_86156a94a0",
|
||||
# "choices": [
|
||||
# {
|
||||
# "index": 0,
|
||||
# "delta": {
|
||||
# "tool_calls": [
|
||||
# {
|
||||
# "index": 0,
|
||||
# "function": {
|
||||
# "arguments": "{\""
|
||||
# }
|
||||
# }
|
||||
# ]
|
||||
# },
|
||||
# "logprobs": null,
|
||||
# "finish_reason": null
|
||||
# }
|
||||
# ]
|
||||
# }
|
||||
|
||||
|
||||
def validate_second_streaming_function_calling_chunk(chunk: ModelResponse):
|
||||
chunk_instance = Chunk2(**chunk.model_dump())
|
||||
|
||||
|
||||
class Delta3(BaseModel):
|
||||
content: Optional[str] = None
|
||||
role: Optional[str] = None
|
||||
function_call: Optional[dict] = None
|
||||
tool_calls: Optional[List] = None
|
||||
|
||||
|
||||
class Choices3(BaseModel):
|
||||
index: int
|
||||
delta: Delta3
|
||||
logprobs: Optional[str]
|
||||
finish_reason: str
|
||||
|
||||
|
||||
class Chunk3(BaseModel):
|
||||
id: str
|
||||
object: str
|
||||
created: int
|
||||
model: str
|
||||
system_fingerprint: str
|
||||
choices: List[Choices3]
|
||||
|
||||
|
||||
def validate_final_streaming_function_calling_chunk(chunk: ModelResponse):
|
||||
chunk_instance = Chunk3(**chunk.model_dump())
|
||||
|
||||
|
||||
def test_azure_streaming_and_function_calling():
|
||||
tools = [
|
||||
{
|
||||
|
|
@ -1690,6 +1886,7 @@ def test_azure_streaming_and_function_calling():
|
|||
)
|
||||
# Add any assertions here to check the response
|
||||
for idx, chunk in enumerate(response):
|
||||
print(f"chunk: {chunk}")
|
||||
if idx == 0:
|
||||
assert (
|
||||
chunk.choices[0].delta.tool_calls[0].function.arguments is not None
|
||||
|
|
@ -1697,40 +1894,69 @@ def test_azure_streaming_and_function_calling():
|
|||
assert isinstance(
|
||||
chunk.choices[0].delta.tool_calls[0].function.arguments, str
|
||||
)
|
||||
validate_first_streaming_function_calling_chunk(chunk=chunk)
|
||||
elif idx == 1:
|
||||
validate_second_streaming_function_calling_chunk(chunk=chunk)
|
||||
elif chunk.choices[0].finish_reason is not None: # last chunk
|
||||
validate_final_streaming_function_calling_chunk(chunk=chunk)
|
||||
|
||||
except Exception as e:
|
||||
pytest.fail(f"Error occurred: {e}")
|
||||
raise e
|
||||
|
||||
|
||||
# test_azure_streaming_and_function_calling()
|
||||
|
||||
|
||||
def test_success_callback_streaming():
|
||||
def success_callback(kwargs, completion_response, start_time, end_time):
|
||||
print(
|
||||
{
|
||||
"success": True,
|
||||
"input": kwargs,
|
||||
"output": completion_response,
|
||||
"start_time": start_time,
|
||||
"end_time": end_time,
|
||||
}
|
||||
@pytest.mark.asyncio
|
||||
async def test_azure_astreaming_and_function_calling():
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_current_weather",
|
||||
"description": "Get the current weather in a given location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {
|
||||
"type": "string",
|
||||
"description": "The city and state, e.g. San Francisco, CA",
|
||||
},
|
||||
"unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
|
||||
},
|
||||
"required": ["location"],
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
messages = [{"role": "user", "content": "What is the weather like in Boston?"}]
|
||||
try:
|
||||
response = await litellm.acompletion(
|
||||
model="azure/gpt-4-nov-release",
|
||||
tools=tools,
|
||||
tool_choice="auto",
|
||||
messages=messages,
|
||||
stream=True,
|
||||
api_base=os.getenv("AZURE_FRANCE_API_BASE"),
|
||||
api_key=os.getenv("AZURE_FRANCE_API_KEY"),
|
||||
api_version="2024-02-15-preview",
|
||||
)
|
||||
# Add any assertions here to check the response
|
||||
idx = 0
|
||||
async for chunk in response:
|
||||
print(f"chunk: {chunk}")
|
||||
if idx == 0:
|
||||
assert (
|
||||
chunk.choices[0].delta.tool_calls[0].function.arguments is not None
|
||||
)
|
||||
assert isinstance(
|
||||
chunk.choices[0].delta.tool_calls[0].function.arguments, str
|
||||
)
|
||||
validate_first_streaming_function_calling_chunk(chunk=chunk)
|
||||
elif idx == 1:
|
||||
validate_second_streaming_function_calling_chunk(chunk=chunk)
|
||||
elif chunk.choices[0].finish_reason is not None: # last chunk
|
||||
validate_final_streaming_function_calling_chunk(chunk=chunk)
|
||||
idx += 1
|
||||
|
||||
litellm.success_callback = [success_callback]
|
||||
|
||||
messages = [{"role": "user", "content": "hello"}]
|
||||
print("TESTING LITELLM COMPLETION CALL")
|
||||
response = litellm.completion(
|
||||
model="j2-light",
|
||||
messages=messages,
|
||||
stream=True,
|
||||
max_tokens=5,
|
||||
)
|
||||
print(response)
|
||||
|
||||
for chunk in response:
|
||||
print(chunk["choices"][0])
|
||||
|
||||
|
||||
# test_success_callback_streaming()
|
||||
except Exception as e:
|
||||
pytest.fail(f"Error occurred: {e}")
|
||||
raise e
|
||||
|
|
|
|||
|
|
@ -376,11 +376,9 @@ class StreamingChoices(OpenAIObject):
|
|||
self.delta = delta
|
||||
else:
|
||||
self.delta = Delta()
|
||||
|
||||
if logprobs is not None:
|
||||
self.logprobs = logprobs
|
||||
if enhancements is not None:
|
||||
self.enhancements = enhancements
|
||||
self.logprobs = logprobs
|
||||
|
||||
def __contains__(self, key):
|
||||
# Define custom behavior for the 'in' operator
|
||||
|
|
@ -820,6 +818,8 @@ class Logging:
|
|||
## DYNAMIC LANGFUSE KEYS ##
|
||||
self.langfuse_public_key = langfuse_public_key
|
||||
self.langfuse_secret = langfuse_secret
|
||||
## TIME TO FIRST TOKEN LOGGING ##
|
||||
self.completion_start_time: Optional[datetime.datetime] = None
|
||||
|
||||
def update_environment_variables(
|
||||
self, model, user, optional_params, litellm_params, **additional_params
|
||||
|
|
@ -840,6 +840,7 @@ class Logging:
|
|||
"user": user,
|
||||
"call_type": str(self.call_type),
|
||||
"litellm_call_id": self.litellm_call_id,
|
||||
"completion_start_time": self.completion_start_time,
|
||||
**self.optional_params,
|
||||
**additional_params,
|
||||
}
|
||||
|
|
@ -1069,6 +1070,11 @@ class Logging:
|
|||
start_time = self.start_time
|
||||
if end_time is None:
|
||||
end_time = datetime.datetime.now()
|
||||
if self.completion_start_time is None:
|
||||
self.completion_start_time = end_time
|
||||
self.model_call_details["completion_start_time"] = (
|
||||
self.completion_start_time
|
||||
)
|
||||
self.model_call_details["log_event_type"] = "successful_api_call"
|
||||
self.model_call_details["end_time"] = end_time
|
||||
self.model_call_details["cache_hit"] = cache_hit
|
||||
|
|
@ -1358,7 +1364,7 @@ class Logging:
|
|||
f"is complete_streaming_response in kwargs: {kwargs.get('complete_streaming_response', None)}"
|
||||
)
|
||||
if complete_streaming_response is None:
|
||||
break
|
||||
continue
|
||||
else:
|
||||
print_verbose("reaches langfuse for streaming logging!")
|
||||
result = kwargs["complete_streaming_response"]
|
||||
|
|
@ -8629,6 +8635,10 @@ class CustomStreamWrapper:
|
|||
model_response.choices[0].finish_reason = response_obj[
|
||||
"finish_reason"
|
||||
]
|
||||
if response_obj.get("original_chunk", None) is not None:
|
||||
model_response.system_fingerprint = getattr(
|
||||
response_obj["original_chunk"], "system_fingerprint", None
|
||||
)
|
||||
if response_obj["logprobs"] is not None:
|
||||
model_response.choices[0].logprobs = response_obj["logprobs"]
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[tool.poetry]
|
||||
name = "litellm"
|
||||
version = "1.26.14.dev1"
|
||||
version = "1.27.1"
|
||||
description = "Library to easily interface with LLM API providers"
|
||||
authors = ["BerriAI"]
|
||||
license = "MIT"
|
||||
|
|
@ -74,7 +74,7 @@ requires = ["poetry-core", "wheel"]
|
|||
build-backend = "poetry.core.masonry.api"
|
||||
|
||||
[tool.commitizen]
|
||||
version = "1.26.14.dev1"
|
||||
version = "1.27.1"
|
||||
version_files = [
|
||||
"pyproject.toml:^version"
|
||||
]
|
||||
|
|
|
|||
|
|
@ -97,10 +97,10 @@ async def test_chat_completion_old_key():
|
|||
"""
|
||||
async with aiohttp.ClientSession() as session:
|
||||
try:
|
||||
key = "sk-yNXvlRO4SxIGG0XnRMYxTw"
|
||||
key = "sk-ecMXHujzUtKCvHcwacdaTw"
|
||||
await chat_completion(session=session, key=key)
|
||||
except Exception as e:
|
||||
key = "sk-2KV0sAElLQqMpLZXdNf3yw" # try diff db key (in case db url is for the other db)
|
||||
key = "sk-ecMXHujzUtKCvHcwacdaTw" # try diff db key (in case db url is for the other db)
|
||||
await chat_completion(session=session, key=key)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,17 +1,26 @@
|
|||
import React, { useState, useEffect } from "react";
|
||||
import ReactMarkdown from "react-markdown";
|
||||
import { Card, Title, Table, TableHead, TableRow, TableCell, TableBody, Grid, Tab,
|
||||
TabGroup,
|
||||
TabList,
|
||||
TabPanel,
|
||||
Metric,
|
||||
Select,
|
||||
SelectItem,
|
||||
TabPanels, } from "@tremor/react";
|
||||
import { modelInfoCall } from "./networking";
|
||||
import {
|
||||
Card,
|
||||
Title,
|
||||
Table,
|
||||
TableHead,
|
||||
TableRow,
|
||||
TableCell,
|
||||
TableBody,
|
||||
Grid,
|
||||
Tab,
|
||||
TabGroup,
|
||||
TabList,
|
||||
TabPanel,
|
||||
Metric,
|
||||
Select,
|
||||
SelectItem,
|
||||
TabPanels,
|
||||
} from "@tremor/react";
|
||||
import { modelAvailableCall } from "./networking";
|
||||
import openai from "openai";
|
||||
import { Prism as SyntaxHighlighter } from 'react-syntax-highlighter';
|
||||
|
||||
import { Prism as SyntaxHighlighter } from "react-syntax-highlighter";
|
||||
|
||||
interface ChatUIProps {
|
||||
accessToken: string | null;
|
||||
|
|
@ -20,12 +29,19 @@ interface ChatUIProps {
|
|||
userID: string | null;
|
||||
}
|
||||
|
||||
async function generateModelResponse(inputMessage: string, updateUI: (chunk: string) => void, selectedModel: string, accessToken: string) {
|
||||
// base url should be the current base_url
|
||||
const isLocal = process.env.NODE_ENV === "development";
|
||||
console.log("isLocal:", isLocal);
|
||||
const proxyBaseUrl = isLocal ? "http://localhost:4000" : window.location.origin;
|
||||
const client = new openai.OpenAI({
|
||||
async function generateModelResponse(
|
||||
inputMessage: string,
|
||||
updateUI: (chunk: string) => void,
|
||||
selectedModel: string,
|
||||
accessToken: string
|
||||
) {
|
||||
// base url should be the current base_url
|
||||
const isLocal = process.env.NODE_ENV === "development";
|
||||
console.log("isLocal:", isLocal);
|
||||
const proxyBaseUrl = isLocal
|
||||
? "http://localhost:4000"
|
||||
: window.location.origin;
|
||||
const client = new openai.OpenAI({
|
||||
apiKey: accessToken, // Replace with your OpenAI API key
|
||||
baseURL: proxyBaseUrl, // Replace with your OpenAI API base URL
|
||||
dangerouslyAllowBrowser: true, // using a temporary litellm proxy key
|
||||
|
|
@ -36,7 +52,7 @@ async function generateModelResponse(inputMessage: string, updateUI: (chunk: str
|
|||
stream: true,
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
role: "user",
|
||||
content: inputMessage,
|
||||
},
|
||||
],
|
||||
|
|
@ -50,138 +66,166 @@ async function generateModelResponse(inputMessage: string, updateUI: (chunk: str
|
|||
}
|
||||
}
|
||||
|
||||
const ChatUI: React.FC<ChatUIProps> = ({ accessToken, token, userRole, userID }) => {
|
||||
const [inputMessage, setInputMessage] = useState("");
|
||||
const [chatHistory, setChatHistory] = useState<any[]>([]);
|
||||
const [selectedModel, setSelectedModel] = useState<string | undefined>(undefined);
|
||||
const [modelInfo, setModelInfo] = useState<any | null>(null); // Declare modelInfo at the component level
|
||||
const ChatUI: React.FC<ChatUIProps> = ({
|
||||
accessToken,
|
||||
token,
|
||||
userRole,
|
||||
userID,
|
||||
}) => {
|
||||
const [inputMessage, setInputMessage] = useState("");
|
||||
const [chatHistory, setChatHistory] = useState<any[]>([]);
|
||||
const [selectedModel, setSelectedModel] = useState<string | undefined>(
|
||||
undefined
|
||||
);
|
||||
const [modelInfo, setModelInfo] = useState<any | null>(null); // Declare modelInfo at the component level
|
||||
|
||||
useEffect(() => {
|
||||
if (!accessToken || !token || !userRole || !userID) {
|
||||
return;
|
||||
useEffect(() => {
|
||||
if (!accessToken || !token || !userRole || !userID) {
|
||||
return;
|
||||
}
|
||||
// Fetch model info and set the default selected model
|
||||
const fetchModelInfo = async () => {
|
||||
const fetchedAvailableModels = await modelAvailableCall(
|
||||
accessToken,
|
||||
userID,
|
||||
userRole
|
||||
);
|
||||
console.log("model_info:", fetchedAvailableModels);
|
||||
|
||||
if (fetchedAvailableModels?.data.length > 0) {
|
||||
setModelInfo(fetchedAvailableModels.data);
|
||||
setSelectedModel(fetchedAvailableModels.data[0].id);
|
||||
}
|
||||
// Fetch model info and set the default selected model
|
||||
const fetchModelInfo = async () => {
|
||||
const fetchedModelInfo = await modelInfoCall(accessToken, userID, userRole);
|
||||
console.log("model_info:", fetchedModelInfo);
|
||||
|
||||
if (fetchedModelInfo?.data.length > 0) {
|
||||
setModelInfo(fetchedModelInfo);
|
||||
setSelectedModel(fetchedModelInfo.data[0].model_name);
|
||||
}
|
||||
};
|
||||
|
||||
fetchModelInfo();
|
||||
}, [accessToken, userID, userRole]);
|
||||
|
||||
const updateUI = (role: string, chunk: string) => {
|
||||
setChatHistory((prevHistory) => {
|
||||
const lastMessage = prevHistory[prevHistory.length - 1];
|
||||
|
||||
if (lastMessage && lastMessage.role === role) {
|
||||
return [
|
||||
...prevHistory.slice(0, prevHistory.length - 1),
|
||||
{ role, content: lastMessage.content + chunk },
|
||||
];
|
||||
} else {
|
||||
return [...prevHistory, { role, content: chunk }];
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
const handleSendMessage = async () => {
|
||||
if (inputMessage.trim() === "") return;
|
||||
fetchModelInfo();
|
||||
}, [accessToken, userID, userRole]);
|
||||
|
||||
if (!accessToken || !token || !userRole || !userID) {
|
||||
return;
|
||||
const updateUI = (role: string, chunk: string) => {
|
||||
setChatHistory((prevHistory) => {
|
||||
const lastMessage = prevHistory[prevHistory.length - 1];
|
||||
|
||||
if (lastMessage && lastMessage.role === role) {
|
||||
return [
|
||||
...prevHistory.slice(0, prevHistory.length - 1),
|
||||
{ role, content: lastMessage.content + chunk },
|
||||
];
|
||||
} else {
|
||||
return [...prevHistory, { role, content: chunk }];
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
setChatHistory((prevHistory) => [
|
||||
...prevHistory,
|
||||
{ role: "user", content: inputMessage },
|
||||
]);
|
||||
const handleSendMessage = async () => {
|
||||
if (inputMessage.trim() === "") return;
|
||||
|
||||
try {
|
||||
if (selectedModel) {
|
||||
await generateModelResponse(inputMessage, (chunk) => updateUI("assistant", chunk), selectedModel, accessToken);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Error fetching model response", error);
|
||||
updateUI("assistant", "Error fetching model response");
|
||||
if (!accessToken || !token || !userRole || !userID) {
|
||||
return;
|
||||
}
|
||||
|
||||
setChatHistory((prevHistory) => [
|
||||
...prevHistory,
|
||||
{ role: "user", content: inputMessage },
|
||||
]);
|
||||
|
||||
try {
|
||||
if (selectedModel) {
|
||||
await generateModelResponse(
|
||||
inputMessage,
|
||||
(chunk) => updateUI("assistant", chunk),
|
||||
selectedModel,
|
||||
accessToken
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Error fetching model response", error);
|
||||
updateUI("assistant", "Error fetching model response");
|
||||
}
|
||||
|
||||
setInputMessage("");
|
||||
};
|
||||
setInputMessage("");
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={{ width: "100%", position: "relative" }}>
|
||||
<Grid className="gap-2 p-10 h-[75vh] w-full">
|
||||
<Card>
|
||||
return (
|
||||
<div style={{ width: "100%", position: "relative" }}>
|
||||
<Grid className="gap-2 p-10 h-[75vh] w-full">
|
||||
<Card>
|
||||
<TabGroup>
|
||||
<TabList className="mt-4">
|
||||
<Tab>Chat</Tab>
|
||||
<Tab>API Reference</Tab>
|
||||
<Tab>Chat</Tab>
|
||||
<Tab>API Reference</Tab>
|
||||
</TabList>
|
||||
|
||||
<TabPanels>
|
||||
<TabPanel>
|
||||
<div>
|
||||
<label>Select Model:</label>
|
||||
<select
|
||||
value={selectedModel || ""}
|
||||
onChange={(e) => setSelectedModel(e.target.value)}
|
||||
<TabPanel>
|
||||
<div>
|
||||
<label>Select Model:</label>
|
||||
<select
|
||||
value={selectedModel || ""}
|
||||
onChange={(e) => setSelectedModel(e.target.value)}
|
||||
>
|
||||
{/* Populate dropdown options from available models */}
|
||||
{modelInfo?.map((element: { id: string }) => (
|
||||
<option key={element.id} value={element.id}>
|
||||
{element.id}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</div>
|
||||
<Table
|
||||
className="mt-5"
|
||||
style={{
|
||||
display: "block",
|
||||
maxHeight: "60vh",
|
||||
overflowY: "auto",
|
||||
}}
|
||||
>
|
||||
<TableHead>
|
||||
<TableRow>
|
||||
<TableCell>
|
||||
<Title>Chat</Title>
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
</TableHead>
|
||||
<TableBody>
|
||||
{chatHistory.map((message, index) => (
|
||||
<TableRow key={index}>
|
||||
<TableCell>{`${message.role}: ${message.content}`}</TableCell>
|
||||
</TableRow>
|
||||
))}
|
||||
</TableBody>
|
||||
</Table>
|
||||
<div
|
||||
className="mt-3"
|
||||
style={{ position: "absolute", bottom: 5, width: "95%" }}
|
||||
>
|
||||
<div className="flex">
|
||||
<input
|
||||
type="text"
|
||||
value={inputMessage}
|
||||
onChange={(e) => setInputMessage(e.target.value)}
|
||||
className="flex-1 p-2 border rounded-md mr-2"
|
||||
placeholder="Type your message..."
|
||||
/>
|
||||
<button
|
||||
onClick={handleSendMessage}
|
||||
className="p-2 bg-blue-500 text-white rounded-md"
|
||||
>
|
||||
{/* Populate dropdown options from available models */}
|
||||
{modelInfo?.data.map((element: { model_name: string }) => (
|
||||
<option key={element.model_name} value={element.model_name}>
|
||||
{element.model_name}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</div>
|
||||
<Table className="mt-5" style={{ display: "block", maxHeight: "60vh", overflowY: "auto" }}>
|
||||
<TableHead>
|
||||
<TableRow>
|
||||
<TableCell>
|
||||
<Title>Chat</Title>
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
</TableHead>
|
||||
<TableBody>
|
||||
{chatHistory.map((message, index) => (
|
||||
<TableRow key={index}>
|
||||
<TableCell>{`${message.role}: ${message.content}`}</TableCell>
|
||||
</TableRow>
|
||||
))}
|
||||
</TableBody>
|
||||
</Table>
|
||||
<div className="mt-3" style={{ position: "absolute", bottom: 5, width: "95%" }}>
|
||||
<div className="flex">
|
||||
<input
|
||||
type="text"
|
||||
value={inputMessage}
|
||||
onChange={(e) => setInputMessage(e.target.value)}
|
||||
className="flex-1 p-2 border rounded-md mr-2"
|
||||
placeholder="Type your message..."
|
||||
/>
|
||||
<button onClick={handleSendMessage} className="p-2 bg-blue-500 text-white rounded-md">
|
||||
Send
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</TabPanel>
|
||||
<TabPanel>
|
||||
<TabGroup>
|
||||
<TabList>
|
||||
<Tab>OpenAI Python SDK</Tab>
|
||||
<Tab>LlamaIndex</Tab>
|
||||
<Tab>Langchain Py</Tab>
|
||||
</TabList>
|
||||
<TabPanels>
|
||||
<TabPanel>
|
||||
|
||||
<SyntaxHighlighter language="python">
|
||||
{`
|
||||
Send
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</TabPanel>
|
||||
<TabPanel>
|
||||
<TabGroup>
|
||||
<TabList>
|
||||
<Tab>OpenAI Python SDK</Tab>
|
||||
<Tab>LlamaIndex</Tab>
|
||||
<Tab>Langchain Py</Tab>
|
||||
</TabList>
|
||||
<TabPanels>
|
||||
<TabPanel>
|
||||
<SyntaxHighlighter language="python">
|
||||
{`
|
||||
import openai
|
||||
client = openai.OpenAI(
|
||||
api_key="your_api_key",
|
||||
|
|
@ -208,12 +252,11 @@ response = client.chat.completions.create(
|
|||
|
||||
print(response)
|
||||
`}
|
||||
</SyntaxHighlighter>
|
||||
</TabPanel>
|
||||
<TabPanel>
|
||||
|
||||
<SyntaxHighlighter language="python">
|
||||
{`
|
||||
</SyntaxHighlighter>
|
||||
</TabPanel>
|
||||
<TabPanel>
|
||||
<SyntaxHighlighter language="python">
|
||||
{`
|
||||
import os, dotenv
|
||||
|
||||
from llama_index.llms import AzureOpenAI
|
||||
|
|
@ -245,12 +288,11 @@ response = query_engine.query("What did the author do growing up?")
|
|||
print(response)
|
||||
|
||||
`}
|
||||
</SyntaxHighlighter>
|
||||
</TabPanel>
|
||||
<TabPanel>
|
||||
|
||||
<SyntaxHighlighter language="python">
|
||||
{`
|
||||
</SyntaxHighlighter>
|
||||
</TabPanel>
|
||||
<TabPanel>
|
||||
<SyntaxHighlighter language="python">
|
||||
{`
|
||||
from langchain.chat_models import ChatOpenAI
|
||||
from langchain.prompts.chat import (
|
||||
ChatPromptTemplate,
|
||||
|
|
@ -286,20 +328,17 @@ response = chat(messages)
|
|||
print(response)
|
||||
|
||||
`}
|
||||
</SyntaxHighlighter>
|
||||
</TabPanel>
|
||||
</TabPanels>
|
||||
</TabGroup>
|
||||
|
||||
</TabPanel>
|
||||
</SyntaxHighlighter>
|
||||
</TabPanel>
|
||||
</TabPanels>
|
||||
</TabGroup>
|
||||
</TabPanel>
|
||||
</TabPanels>
|
||||
</TabGroup>
|
||||
</Card>
|
||||
</Grid>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
</TabGroup>
|
||||
</Card>
|
||||
</Grid>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
|
||||
|
||||
export default ChatUI;
|
||||
export default ChatUI;
|
||||
|
|
|
|||
|
|
@ -1,10 +1,17 @@
|
|||
|
||||
"use client";
|
||||
|
||||
import React, { useState, useEffect, useRef } from "react";
|
||||
import { Button, TextInput, Grid, Col } from "@tremor/react";
|
||||
import { Card, Metric, Text } from "@tremor/react";
|
||||
import { Button as Button2, Modal, Form, Input, InputNumber, Select, message } from "antd";
|
||||
import {
|
||||
Button as Button2,
|
||||
Modal,
|
||||
Form,
|
||||
Input,
|
||||
InputNumber,
|
||||
Select,
|
||||
message,
|
||||
} from "antd";
|
||||
import { keyCreateCall } from "./networking";
|
||||
|
||||
const { Option } = Select;
|
||||
|
|
@ -50,13 +57,12 @@ const CreateKey: React.FC<CreateKeyProps> = ({
|
|||
setApiKey(response["key"]);
|
||||
message.success("API Key Created");
|
||||
form.resetFields();
|
||||
localStorage.removeItem("userData" + userID)
|
||||
localStorage.removeItem("userData" + userID);
|
||||
} catch (error) {
|
||||
console.error("Error creating the key:", error);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
return (
|
||||
<div>
|
||||
<Button className="mx-auto" onClick={() => setIsModalVisible(true)}>
|
||||
|
|
@ -70,98 +76,70 @@ const CreateKey: React.FC<CreateKeyProps> = ({
|
|||
onOk={handleOk}
|
||||
onCancel={handleCancel}
|
||||
>
|
||||
<Form form={form} onFinish={handleCreate} labelCol={{ span: 8 }} wrapperCol={{ span: 16 }} labelAlign="left">
|
||||
{userRole === 'App Owner' || userRole === 'Admin' ? (
|
||||
<Form
|
||||
form={form}
|
||||
onFinish={handleCreate}
|
||||
labelCol={{ span: 8 }}
|
||||
wrapperCol={{ span: 16 }}
|
||||
labelAlign="left"
|
||||
>
|
||||
{userRole === "App Owner" || userRole === "Admin" ? (
|
||||
<>
|
||||
|
||||
<Form.Item
|
||||
label="Key Name"
|
||||
name="key_alias"
|
||||
>
|
||||
<Input />
|
||||
</Form.Item>
|
||||
<Form.Item
|
||||
label="Team ID"
|
||||
name="team_id"
|
||||
>
|
||||
<Input placeholder="ai_team" />
|
||||
</Form.Item>
|
||||
<Form.Item
|
||||
label="Models"
|
||||
name="models"
|
||||
>
|
||||
<Select
|
||||
mode="multiple"
|
||||
placeholder="Select models"
|
||||
style={{ width: '100%' }}
|
||||
>
|
||||
{userModels.map((model) => (
|
||||
<Option key={model} value={model}>
|
||||
{model}
|
||||
</Option>
|
||||
))}
|
||||
</Select>
|
||||
</Form.Item>
|
||||
<Form.Item
|
||||
label="Max Budget (USD)"
|
||||
name="max_budget"
|
||||
>
|
||||
<InputNumber step={0.01} precision={2} width={200}/>
|
||||
</Form.Item>
|
||||
<Form.Item
|
||||
label="Tokens per minute Limit (TPM)"
|
||||
name="tpm_limit"
|
||||
>
|
||||
<InputNumber step={1} width={400}/>
|
||||
</Form.Item>
|
||||
<Form.Item
|
||||
label="Requests per minute Limit (RPM)"
|
||||
name="rpm_limit"
|
||||
>
|
||||
<InputNumber step={1} width={400}/>
|
||||
</Form.Item>
|
||||
<Form.Item
|
||||
label="Duration (eg: 30s, 30h, 30d)"
|
||||
name="duration"
|
||||
>
|
||||
<Input />
|
||||
</Form.Item>
|
||||
<Form.Item
|
||||
label="Metadata"
|
||||
name="metadata"
|
||||
>
|
||||
<Input.TextArea rows={4} placeholder="Enter metadata as JSON" />
|
||||
</Form.Item>
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<Form.Item label="Key Name" name="key_alias">
|
||||
<Input />
|
||||
</Form.Item>
|
||||
<Form.Item label="Team ID" name="team_id">
|
||||
<Input placeholder="ai_team" />
|
||||
</Form.Item>
|
||||
<Form.Item label="Models" name="models">
|
||||
<Select
|
||||
mode="multiple"
|
||||
placeholder="Select models"
|
||||
style={{ width: "100%" }}
|
||||
>
|
||||
{userModels.map((model) => (
|
||||
<Option key={model} value={model}>
|
||||
{model}
|
||||
</Option>
|
||||
))}
|
||||
</Select>
|
||||
</Form.Item>
|
||||
<Form.Item label="Max Budget (USD)" name="max_budget">
|
||||
<InputNumber step={0.01} precision={2} width={200} />
|
||||
</Form.Item>
|
||||
<Form.Item label="Tokens per minute Limit (TPM)" name="tpm_limit">
|
||||
<InputNumber step={1} width={400} />
|
||||
</Form.Item>
|
||||
<Form.Item
|
||||
label="Key Name"
|
||||
name="key_alias"
|
||||
>
|
||||
<Input />
|
||||
</Form.Item>
|
||||
<Form.Item
|
||||
label="Team ID (Contact Group)"
|
||||
name="team_id"
|
||||
>
|
||||
<Input placeholder="ai_team" />
|
||||
</Form.Item>
|
||||
label="Requests per minute Limit (RPM)"
|
||||
name="rpm_limit"
|
||||
>
|
||||
<InputNumber step={1} width={400} />
|
||||
</Form.Item>
|
||||
<Form.Item label="Duration (eg: 30s, 30h, 30d)" name="duration">
|
||||
<Input />
|
||||
</Form.Item>
|
||||
<Form.Item label="Metadata" name="metadata">
|
||||
<Input.TextArea rows={4} placeholder="Enter metadata as JSON" />
|
||||
</Form.Item>
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<Form.Item label="Key Name" name="key_alias">
|
||||
<Input />
|
||||
</Form.Item>
|
||||
<Form.Item label="Team ID (Contact Group)" name="team_id">
|
||||
<Input placeholder="ai_team" />
|
||||
</Form.Item>
|
||||
|
||||
<Form.Item
|
||||
label="Description"
|
||||
name="description"
|
||||
>
|
||||
<Input.TextArea placeholder="Enter description" rows={4}/>
|
||||
</Form.Item>
|
||||
</>
|
||||
)
|
||||
}
|
||||
<div style={{ textAlign: 'right', marginTop: '10px' }}>
|
||||
<Button2 htmlType="submit">
|
||||
Create Key
|
||||
</Button2>
|
||||
</div>
|
||||
<Form.Item label="Description" name="description">
|
||||
<Input.TextArea placeholder="Enter description" rows={4} />
|
||||
</Form.Item>
|
||||
</>
|
||||
)}
|
||||
<div style={{ textAlign: "right", marginTop: "10px" }}>
|
||||
<Button2 htmlType="submit">Create Key</Button2>
|
||||
</div>
|
||||
</Form>
|
||||
</Modal>
|
||||
{apiKey && (
|
||||
|
|
@ -173,22 +151,22 @@ const CreateKey: React.FC<CreateKeyProps> = ({
|
|||
footer={null}
|
||||
>
|
||||
<Grid numItems={1} className="gap-2 w-full">
|
||||
<Col numColSpan={1}>
|
||||
<p>
|
||||
Please save this secret key somewhere safe and accessible. For
|
||||
security reasons, <b>you will not be able to view it again</b>{" "}
|
||||
through your LiteLLM account. If you lose this secret key, you will
|
||||
need to generate a new one.
|
||||
</p>
|
||||
</Col>
|
||||
<Col numColSpan={1}>
|
||||
{apiKey != null ? (
|
||||
<Text>API Key: {apiKey}</Text>
|
||||
) : (
|
||||
<Text>Key being created, this might take 30s</Text>
|
||||
)}
|
||||
</Col>
|
||||
</Grid>
|
||||
<Col numColSpan={1}>
|
||||
<p>
|
||||
Please save this secret key somewhere safe and accessible. For
|
||||
security reasons, <b>you will not be able to view it again</b>{" "}
|
||||
through your LiteLLM account. If you lose this secret key, you
|
||||
will need to generate a new one.
|
||||
</p>
|
||||
</Col>
|
||||
<Col numColSpan={1}>
|
||||
{apiKey != null ? (
|
||||
<Text>API Key: {apiKey}</Text>
|
||||
) : (
|
||||
<Text>Key being created, this might take 30s</Text>
|
||||
)}
|
||||
</Col>
|
||||
</Grid>
|
||||
</Modal>
|
||||
)}
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import React, { useState, useEffect } from "react";
|
||||
import { Button, Modal, Form, Input, message, Select, InputNumber } from "antd";
|
||||
import { Button as Button2 } from "@tremor/react";
|
||||
import { userCreateCall, modelInfoCall } from "./networking";
|
||||
import { userCreateCall, modelAvailableCall } from "./networking";
|
||||
const { Option } = Select;
|
||||
|
||||
interface CreateuserProps {
|
||||
|
|
@ -20,12 +20,16 @@ const Createuser: React.FC<CreateuserProps> = ({ userID, accessToken }) => {
|
|||
const fetchData = async () => {
|
||||
try {
|
||||
const userRole = "any"; // You may need to get the user role dynamically
|
||||
const modelDataResponse = await modelInfoCall(accessToken, userID, userRole);
|
||||
const modelDataResponse = await modelAvailableCall(
|
||||
accessToken,
|
||||
userID,
|
||||
userRole
|
||||
);
|
||||
// Assuming modelDataResponse.data contains an array of model objects with a 'model_name' property
|
||||
const availableModels = [];
|
||||
for (let i = 0; i < modelDataResponse.data.length; i++) {
|
||||
const model = modelDataResponse.data[i];
|
||||
availableModels.push(model.model_name);
|
||||
const model = modelDataResponse.data[i];
|
||||
availableModels.push(model.id);
|
||||
}
|
||||
console.log("Model data response:", modelDataResponse.data);
|
||||
console.log("Available models:", availableModels);
|
||||
|
|
@ -79,77 +83,51 @@ const Createuser: React.FC<CreateuserProps> = ({ userID, accessToken }) => {
|
|||
onOk={handleOk}
|
||||
onCancel={handleCancel}
|
||||
>
|
||||
<Form form={form} onFinish={handleCreate} labelCol={{ span: 8 }} wrapperCol={{ span: 16 }} labelAlign="left">
|
||||
<Form.Item
|
||||
label="User ID"
|
||||
name="user_id"
|
||||
>
|
||||
<Form
|
||||
form={form}
|
||||
onFinish={handleCreate}
|
||||
labelCol={{ span: 8 }}
|
||||
wrapperCol={{ span: 16 }}
|
||||
labelAlign="left"
|
||||
>
|
||||
<Form.Item label="User ID" name="user_id">
|
||||
<Input placeholder="Enter User ID" />
|
||||
</Form.Item>
|
||||
<Form.Item
|
||||
label="Team ID"
|
||||
name="team_id"
|
||||
>
|
||||
<Input placeholder="ai_team" />
|
||||
</Form.Item>
|
||||
<Form.Item
|
||||
label="Models"
|
||||
name="models"
|
||||
>
|
||||
<Form.Item label="Team ID" name="team_id">
|
||||
<Input placeholder="ai_team" />
|
||||
</Form.Item>
|
||||
<Form.Item label="Models" name="models">
|
||||
<Select
|
||||
mode="multiple"
|
||||
placeholder="Select models"
|
||||
style={{ width: '100%' }}
|
||||
mode="multiple"
|
||||
placeholder="Select models"
|
||||
style={{ width: "100%" }}
|
||||
>
|
||||
{userModels.map((model) => (
|
||||
{userModels.map((model) => (
|
||||
<Option key={model} value={model}>
|
||||
{model}
|
||||
{model}
|
||||
</Option>
|
||||
))}
|
||||
))}
|
||||
</Select>
|
||||
</Form.Item>
|
||||
</Form.Item>
|
||||
|
||||
|
||||
<Form.Item
|
||||
label="Max Budget (USD)"
|
||||
name="max_budget"
|
||||
>
|
||||
<InputNumber step={0.01} precision={2} width={200}/>
|
||||
|
||||
</Form.Item>
|
||||
<Form.Item
|
||||
label="Tokens per minute Limit (TPM)"
|
||||
name="tpm_limit"
|
||||
>
|
||||
<InputNumber step={1} width={400}/>
|
||||
|
||||
</Form.Item>
|
||||
<Form.Item
|
||||
label="Requests per minute Limit (RPM)"
|
||||
name="rpm_limit"
|
||||
>
|
||||
<InputNumber step={1} width={400}/>
|
||||
|
||||
</Form.Item>
|
||||
<Form.Item
|
||||
label="Duration (eg: 30s, 30h, 30d)"
|
||||
name="duration"
|
||||
>
|
||||
<Input />
|
||||
|
||||
</Form.Item>
|
||||
<Form.Item
|
||||
label="Metadata"
|
||||
name="metadata"
|
||||
>
|
||||
<Input.TextArea rows={4} placeholder="Enter metadata as JSON" />
|
||||
|
||||
</Form.Item>
|
||||
<div style={{ textAlign: 'right', marginTop: '10px' }}>
|
||||
<Button htmlType="submit">
|
||||
Create User
|
||||
</Button>
|
||||
</div>
|
||||
<Form.Item label="Max Budget (USD)" name="max_budget">
|
||||
<InputNumber step={0.01} precision={2} width={200} />
|
||||
</Form.Item>
|
||||
<Form.Item label="Tokens per minute Limit (TPM)" name="tpm_limit">
|
||||
<InputNumber step={1} width={400} />
|
||||
</Form.Item>
|
||||
<Form.Item label="Requests per minute Limit (RPM)" name="rpm_limit">
|
||||
<InputNumber step={1} width={400} />
|
||||
</Form.Item>
|
||||
<Form.Item label="Duration (eg: 30s, 30h, 30d)" name="duration">
|
||||
<Input />
|
||||
</Form.Item>
|
||||
<Form.Item label="Metadata" name="metadata">
|
||||
<Input.TextArea rows={4} placeholder="Enter metadata as JSON" />
|
||||
</Form.Item>
|
||||
<div style={{ textAlign: "right", marginTop: "10px" }}>
|
||||
<Button htmlType="submit">Create User</Button>
|
||||
</div>
|
||||
</Form>
|
||||
</Modal>
|
||||
{apiuser && (
|
||||
|
|
@ -162,11 +140,15 @@ const Createuser: React.FC<CreateuserProps> = ({ userID, accessToken }) => {
|
|||
>
|
||||
<p>
|
||||
Please save this secret user somewhere safe and accessible. For
|
||||
security reasons, <b>you will not be able to view it again</b> through
|
||||
your LiteLLM account. If you lose this secret user, you will need to
|
||||
generate a new one.
|
||||
security reasons, <b>you will not be able to view it again</b>{" "}
|
||||
through your LiteLLM account. If you lose this secret user, you will
|
||||
need to generate a new one.
|
||||
</p>
|
||||
<p>
|
||||
{apiuser != null
|
||||
? `API user: ${apiuser}`
|
||||
: "User being created, this might take 30s"}
|
||||
</p>
|
||||
<p>{apiuser != null ? `API user: ${apiuser}` : "User being created, this might take 30s"}</p>
|
||||
</Modal>
|
||||
)}
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -69,7 +69,6 @@ export const keyCreateCall = async (
|
|||
}
|
||||
};
|
||||
|
||||
|
||||
export const userCreateCall = async (
|
||||
accessToken: string,
|
||||
userID: string,
|
||||
|
|
@ -133,7 +132,6 @@ export const userCreateCall = async (
|
|||
}
|
||||
};
|
||||
|
||||
|
||||
export const keyDeleteCall = async (accessToken: String, user_key: String) => {
|
||||
try {
|
||||
const url = proxyBaseUrl ? `${proxyBaseUrl}/key/delete` : `/key/delete`;
|
||||
|
|
@ -207,13 +205,14 @@ export const userInfoCall = async (
|
|||
}
|
||||
};
|
||||
|
||||
|
||||
|
||||
export const modelInfoCall = async (
|
||||
accessToken: String,
|
||||
userID: String,
|
||||
userRole: String
|
||||
) => {
|
||||
/**
|
||||
* Get all models on proxy
|
||||
*/
|
||||
try {
|
||||
let url = proxyBaseUrl ? `${proxyBaseUrl}/v2/model/info` : `/v2/model/info`;
|
||||
|
||||
|
|
@ -242,6 +241,42 @@ export const modelInfoCall = async (
|
|||
}
|
||||
};
|
||||
|
||||
export const modelAvailableCall = async (
|
||||
accessToken: String,
|
||||
userID: String,
|
||||
userRole: String
|
||||
) => {
|
||||
/**
|
||||
* Get all the models user has access to
|
||||
*/
|
||||
try {
|
||||
let url = proxyBaseUrl ? `${proxyBaseUrl}/models` : `/models`;
|
||||
|
||||
message.info("Requesting model data");
|
||||
const response = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error(errorData);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
message.info("Received model data");
|
||||
return data;
|
||||
// Handle success - you might want to update some state or UI based on the created key
|
||||
} catch (error) {
|
||||
console.error("Failed to create key:", error);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
export const keySpendLogsCall = async (accessToken: String, token: String) => {
|
||||
try {
|
||||
const url = proxyBaseUrl ? `${proxyBaseUrl}/spend/logs` : `/spend/logs`;
|
||||
|
|
@ -363,12 +398,16 @@ export const spendUsersCall = async (accessToken: String, userID: String) => {
|
|||
}
|
||||
};
|
||||
|
||||
|
||||
|
||||
|
||||
export const userRequestModelCall = async (accessToken: String, model: String, UserID: String, justification: String) => {
|
||||
export const userRequestModelCall = async (
|
||||
accessToken: String,
|
||||
model: String,
|
||||
UserID: String,
|
||||
justification: String
|
||||
) => {
|
||||
try {
|
||||
const url = proxyBaseUrl ? `${proxyBaseUrl}/user/request_model` : `/user/request_model`;
|
||||
const url = proxyBaseUrl
|
||||
? `${proxyBaseUrl}/user/request_model`
|
||||
: `/user/request_model`;
|
||||
const response = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
|
|
@ -398,10 +437,11 @@ export const userRequestModelCall = async (accessToken: String, model: String, U
|
|||
}
|
||||
};
|
||||
|
||||
|
||||
export const userGetRequesedtModelsCall = async (accessToken: String) => {
|
||||
try {
|
||||
const url = proxyBaseUrl ? `${proxyBaseUrl}/user/get_requests` : `/user/get_requests`;
|
||||
const url = proxyBaseUrl
|
||||
? `${proxyBaseUrl}/user/get_requests`
|
||||
: `/user/get_requests`;
|
||||
console.log("in userGetRequesedtModelsCall:", url);
|
||||
const response = await fetch(url, {
|
||||
method: "GET",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
"use client";
|
||||
import React, { useState, useEffect } from "react";
|
||||
import { userInfoCall, modelInfoCall } from "./networking";
|
||||
import { userInfoCall, modelAvailableCall } from "./networking";
|
||||
import { Grid, Col, Card, Text } from "@tremor/react";
|
||||
import CreateKey from "./create_key_button";
|
||||
import ViewKeyTable from "./view_key_table";
|
||||
|
|
@ -48,10 +48,9 @@ const UserDashboard: React.FC<UserDashboardProps> = ({
|
|||
const token = searchParams.get("token");
|
||||
const [accessToken, setAccessToken] = useState<string | null>(null);
|
||||
const [userModels, setUserModels] = useState<string[]>([]);
|
||||
|
||||
// check if window is not undefined
|
||||
if (typeof window !== "undefined") {
|
||||
window.addEventListener('beforeunload', function() {
|
||||
window.addEventListener("beforeunload", function () {
|
||||
// Clear session storage
|
||||
sessionStorage.clear();
|
||||
});
|
||||
|
|
@ -78,7 +77,6 @@ const UserDashboard: React.FC<UserDashboardProps> = ({
|
|||
|
||||
// Moved useEffect inside the component and used a condition to run fetch only if the params are available
|
||||
useEffect(() => {
|
||||
|
||||
if (token) {
|
||||
const decoded = jwtDecode(token) as { [key: string]: any };
|
||||
if (decoded) {
|
||||
|
|
@ -109,32 +107,39 @@ const UserDashboard: React.FC<UserDashboardProps> = ({
|
|||
const cachedUserModels = sessionStorage.getItem("userModels" + userID);
|
||||
if (cachedUserModels) {
|
||||
setUserModels(JSON.parse(cachedUserModels));
|
||||
|
||||
} else {
|
||||
const fetchData = async () => {
|
||||
try {
|
||||
const response = await userInfoCall(accessToken, userID, userRole);
|
||||
setUserSpendData(response["user_info"]);
|
||||
setData(response["keys"]); // Assuming this is the correct path to your data
|
||||
sessionStorage.setItem("userData" + userID, JSON.stringify(response["keys"]));
|
||||
sessionStorage.setItem(
|
||||
"userData" + userID,
|
||||
JSON.stringify(response["keys"])
|
||||
);
|
||||
sessionStorage.setItem(
|
||||
"userSpendData" + userID,
|
||||
JSON.stringify(response["user_info"])
|
||||
);
|
||||
|
||||
const model_info = await modelInfoCall(accessToken, userID, userRole);
|
||||
console.log("model_info:", model_info);
|
||||
const model_available = await modelAvailableCall(
|
||||
accessToken,
|
||||
userID,
|
||||
userRole
|
||||
);
|
||||
// loop through model_info["data"] and create an array of element.model_name
|
||||
let available_model_names = model_info["data"].filter((element: { model_name: string; user_access: boolean }) => element.user_access === true).map((element: { model_name: string; }) => element.model_name);
|
||||
let available_model_names = model_available["data"].map(
|
||||
(element: { id: string }) => element.id
|
||||
);
|
||||
console.log("available_model_names:", available_model_names);
|
||||
setUserModels(available_model_names);
|
||||
|
||||
console.log("userModels:", userModels);
|
||||
|
||||
sessionStorage.setItem("userModels" + userID, JSON.stringify(available_model_names));
|
||||
|
||||
|
||||
|
||||
sessionStorage.setItem(
|
||||
"userModels" + userID,
|
||||
JSON.stringify(available_model_names)
|
||||
);
|
||||
} catch (error) {
|
||||
console.error("There was an error fetching the data", error);
|
||||
// Optionally, update your UI to reflect the error state here as well
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue