mirror of
https://github.com/BerriAI/litellm.git
synced 2025-04-26 03:04:13 +00:00
(fix) LiteLLM Proxy fix GET /files/{file_id:path}/content"
endpoint (#7342)
* fix order of get_file_content * update e2 files tests * add e2 batches endpoint testing * update config.yml * write content to file * use correct oai_misc_config * fixes for openai batches endpoint testing * remove extra out file * fix input.jsonl
This commit is contained in:
parent
c78893723f
commit
b90b98b88f
10 changed files with 414 additions and 135 deletions
|
@ -998,6 +998,124 @@ jobs:
|
||||||
python -m pytest -s -vv tests/*.py -x --junitxml=test-results/junit.xml --durations=5 --ignore=tests/otel_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests
|
python -m pytest -s -vv tests/*.py -x --junitxml=test-results/junit.xml --durations=5 --ignore=tests/otel_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests
|
||||||
no_output_timeout: 120m
|
no_output_timeout: 120m
|
||||||
|
|
||||||
|
# Store test results
|
||||||
|
- store_test_results:
|
||||||
|
path: test-results
|
||||||
|
e2e_openai_misc_endpoints:
|
||||||
|
machine:
|
||||||
|
image: ubuntu-2204:2023.10.1
|
||||||
|
resource_class: xlarge
|
||||||
|
working_directory: ~/project
|
||||||
|
steps:
|
||||||
|
- checkout
|
||||||
|
- run:
|
||||||
|
name: Install Docker CLI (In case it's not already installed)
|
||||||
|
command: |
|
||||||
|
sudo apt-get update
|
||||||
|
sudo apt-get install -y docker-ce docker-ce-cli containerd.io
|
||||||
|
- run:
|
||||||
|
name: Install Python 3.9
|
||||||
|
command: |
|
||||||
|
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
|
||||||
|
bash miniconda.sh -b -p $HOME/miniconda
|
||||||
|
export PATH="$HOME/miniconda/bin:$PATH"
|
||||||
|
conda init bash
|
||||||
|
source ~/.bashrc
|
||||||
|
conda create -n myenv python=3.9 -y
|
||||||
|
conda activate myenv
|
||||||
|
python --version
|
||||||
|
- run:
|
||||||
|
name: Install Dependencies
|
||||||
|
command: |
|
||||||
|
pip install "pytest==7.3.1"
|
||||||
|
pip install "pytest-asyncio==0.21.1"
|
||||||
|
pip install aiohttp
|
||||||
|
python -m pip install --upgrade pip
|
||||||
|
python -m pip install -r .circleci/requirements.txt
|
||||||
|
pip install "pytest==7.3.1"
|
||||||
|
pip install "pytest-retry==1.6.3"
|
||||||
|
pip install "pytest-mock==3.12.0"
|
||||||
|
pip install "pytest-asyncio==0.21.1"
|
||||||
|
pip install mypy
|
||||||
|
pip install "jsonlines==4.0.0"
|
||||||
|
pip install "google-generativeai==0.3.2"
|
||||||
|
pip install "google-cloud-aiplatform==1.43.0"
|
||||||
|
pip install pyarrow
|
||||||
|
pip install "boto3==1.34.34"
|
||||||
|
pip install "aioboto3==12.3.0"
|
||||||
|
pip install langchain
|
||||||
|
pip install "langfuse>=2.0.0"
|
||||||
|
pip install "logfire==0.29.0"
|
||||||
|
pip install numpydoc
|
||||||
|
pip install prisma
|
||||||
|
pip install fastapi
|
||||||
|
pip install jsonschema
|
||||||
|
pip install "httpx==0.24.1"
|
||||||
|
pip install "gunicorn==21.2.0"
|
||||||
|
pip install "anyio==3.7.1"
|
||||||
|
pip install "aiodynamo==23.10.1"
|
||||||
|
pip install "asyncio==3.4.3"
|
||||||
|
pip install "PyGithub==1.59.1"
|
||||||
|
pip install "openai==1.54.0 "
|
||||||
|
# Run pytest and generate JUnit XML report
|
||||||
|
- run:
|
||||||
|
name: Build Docker image
|
||||||
|
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
|
||||||
|
- run:
|
||||||
|
name: Run Docker container
|
||||||
|
command: |
|
||||||
|
docker run -d \
|
||||||
|
-p 4000:4000 \
|
||||||
|
-e DATABASE_URL=$PROXY_DATABASE_URL \
|
||||||
|
-e AZURE_API_KEY=$AZURE_API_KEY \
|
||||||
|
-e REDIS_HOST=$REDIS_HOST \
|
||||||
|
-e REDIS_PASSWORD=$REDIS_PASSWORD \
|
||||||
|
-e REDIS_PORT=$REDIS_PORT \
|
||||||
|
-e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \
|
||||||
|
-e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \
|
||||||
|
-e MISTRAL_API_KEY=$MISTRAL_API_KEY \
|
||||||
|
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
|
||||||
|
-e GROQ_API_KEY=$GROQ_API_KEY \
|
||||||
|
-e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
|
||||||
|
-e COHERE_API_KEY=$COHERE_API_KEY \
|
||||||
|
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
|
||||||
|
-e AWS_REGION_NAME=$AWS_REGION_NAME \
|
||||||
|
-e AUTO_INFER_REGION=True \
|
||||||
|
-e OPENAI_API_KEY=$OPENAI_API_KEY \
|
||||||
|
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
||||||
|
-e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \
|
||||||
|
-e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \
|
||||||
|
-e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \
|
||||||
|
-e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \
|
||||||
|
--name my-app \
|
||||||
|
-v $(pwd)/litellm/proxy/example_config_yaml/oai_misc_config.yaml:/app/config.yaml \
|
||||||
|
my-app:latest \
|
||||||
|
--config /app/config.yaml \
|
||||||
|
--port 4000 \
|
||||||
|
--detailed_debug \
|
||||||
|
- run:
|
||||||
|
name: Install curl and dockerize
|
||||||
|
command: |
|
||||||
|
sudo apt-get update
|
||||||
|
sudo apt-get install -y curl
|
||||||
|
sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
|
||||||
|
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
|
||||||
|
sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
|
||||||
|
- run:
|
||||||
|
name: Start outputting logs
|
||||||
|
command: docker logs -f my-app
|
||||||
|
background: true
|
||||||
|
- run:
|
||||||
|
name: Wait for app to be ready
|
||||||
|
command: dockerize -wait http://localhost:4000 -timeout 5m
|
||||||
|
- run:
|
||||||
|
name: Run tests
|
||||||
|
command: |
|
||||||
|
pwd
|
||||||
|
ls
|
||||||
|
python -m pytest -s -vv tests/openai_misc_endpoints_tests --junitxml=test-results/junit.xml --durations=5
|
||||||
|
no_output_timeout: 120m
|
||||||
|
|
||||||
# Store test results
|
# Store test results
|
||||||
- store_test_results:
|
- store_test_results:
|
||||||
path: test-results
|
path: test-results
|
||||||
|
@ -1572,6 +1690,12 @@ workflows:
|
||||||
only:
|
only:
|
||||||
- main
|
- main
|
||||||
- /litellm_.*/
|
- /litellm_.*/
|
||||||
|
- e2e_openai_misc_endpoints:
|
||||||
|
filters:
|
||||||
|
branches:
|
||||||
|
only:
|
||||||
|
- main
|
||||||
|
- /litellm_.*/
|
||||||
- proxy_logging_guardrails_model_info_tests:
|
- proxy_logging_guardrails_model_info_tests:
|
||||||
filters:
|
filters:
|
||||||
branches:
|
branches:
|
||||||
|
@ -1655,6 +1779,7 @@ workflows:
|
||||||
requires:
|
requires:
|
||||||
- local_testing
|
- local_testing
|
||||||
- build_and_test
|
- build_and_test
|
||||||
|
- e2e_openai_misc_endpoints
|
||||||
- load_testing
|
- load_testing
|
||||||
- test_bad_database_url
|
- test_bad_database_url
|
||||||
- llm_translation_testing
|
- llm_translation_testing
|
||||||
|
|
62
litellm/proxy/example_config_yaml/oai_misc_config.yaml
Normal file
62
litellm/proxy/example_config_yaml/oai_misc_config.yaml
Normal file
|
@ -0,0 +1,62 @@
|
||||||
|
model_list:
|
||||||
|
- model_name: gpt-3.5-turbo-end-user-test
|
||||||
|
litellm_params:
|
||||||
|
model: gpt-3.5-turbo
|
||||||
|
region_name: "eu"
|
||||||
|
model_info:
|
||||||
|
id: "1"
|
||||||
|
- model_name: "*"
|
||||||
|
litellm_params:
|
||||||
|
model: openai/*
|
||||||
|
api_key: os.environ/OPENAI_API_KEY
|
||||||
|
# provider specific wildcard routing
|
||||||
|
- model_name: "anthropic/*"
|
||||||
|
litellm_params:
|
||||||
|
model: "anthropic/*"
|
||||||
|
api_key: os.environ/ANTHROPIC_API_KEY
|
||||||
|
- model_name: "groq/*"
|
||||||
|
litellm_params:
|
||||||
|
model: "groq/*"
|
||||||
|
api_key: os.environ/GROQ_API_KEY
|
||||||
|
litellm_settings:
|
||||||
|
# set_verbose: True # Uncomment this if you want to see verbose logs; not recommended in production
|
||||||
|
drop_params: True
|
||||||
|
# max_budget: 100
|
||||||
|
# budget_duration: 30d
|
||||||
|
num_retries: 5
|
||||||
|
request_timeout: 600
|
||||||
|
telemetry: False
|
||||||
|
context_window_fallbacks: [{"gpt-3.5-turbo": ["gpt-3.5-turbo-large"]}]
|
||||||
|
default_team_settings:
|
||||||
|
- team_id: team-1
|
||||||
|
success_callback: ["langfuse"]
|
||||||
|
failure_callback: ["langfuse"]
|
||||||
|
langfuse_public_key: os.environ/LANGFUSE_PROJECT1_PUBLIC # Project 1
|
||||||
|
langfuse_secret: os.environ/LANGFUSE_PROJECT1_SECRET # Project 1
|
||||||
|
- team_id: team-2
|
||||||
|
success_callback: ["langfuse"]
|
||||||
|
failure_callback: ["langfuse"]
|
||||||
|
langfuse_public_key: os.environ/LANGFUSE_PROJECT2_PUBLIC # Project 2
|
||||||
|
langfuse_secret: os.environ/LANGFUSE_PROJECT2_SECRET # Project 2
|
||||||
|
langfuse_host: https://us.cloud.langfuse.com
|
||||||
|
|
||||||
|
# For /fine_tuning/jobs endpoints
|
||||||
|
finetune_settings:
|
||||||
|
- custom_llm_provider: azure
|
||||||
|
api_base: https://exampleopenaiendpoint-production.up.railway.app
|
||||||
|
api_key: fake-key
|
||||||
|
api_version: "2023-03-15-preview"
|
||||||
|
- custom_llm_provider: openai
|
||||||
|
api_key: os.environ/OPENAI_API_KEY
|
||||||
|
|
||||||
|
# for /files endpoints
|
||||||
|
files_settings:
|
||||||
|
- custom_llm_provider: azure
|
||||||
|
api_base: https://exampleopenaiendpoint-production.up.railway.app
|
||||||
|
api_key: fake-key
|
||||||
|
api_version: "2023-03-15-preview"
|
||||||
|
- custom_llm_provider: openai
|
||||||
|
api_key: os.environ/OPENAI_API_KEY
|
||||||
|
|
||||||
|
general_settings:
|
||||||
|
master_key: sk-1234 # [OPTIONAL] Use to enforce auth on proxy. See - https://docs.litellm.ai/docs/proxy/virtual_keys
|
|
@ -265,6 +265,131 @@ async def create_file(
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/{provider}/v1/files/{file_id:path}/content",
|
||||||
|
dependencies=[Depends(user_api_key_auth)],
|
||||||
|
tags=["files"],
|
||||||
|
)
|
||||||
|
@router.get(
|
||||||
|
"/v1/files/{file_id:path}/content",
|
||||||
|
dependencies=[Depends(user_api_key_auth)],
|
||||||
|
tags=["files"],
|
||||||
|
)
|
||||||
|
@router.get(
|
||||||
|
"/files/{file_id:path}/content",
|
||||||
|
dependencies=[Depends(user_api_key_auth)],
|
||||||
|
tags=["files"],
|
||||||
|
)
|
||||||
|
async def get_file_content(
|
||||||
|
request: Request,
|
||||||
|
fastapi_response: Response,
|
||||||
|
file_id: str,
|
||||||
|
provider: Optional[str] = None,
|
||||||
|
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Returns information about a specific file. that can be used across - Assistants API, Batch API
|
||||||
|
This is the equivalent of GET https://api.openai.com/v1/files/{file_id}/content
|
||||||
|
|
||||||
|
Supports Identical Params as: https://platform.openai.com/docs/api-reference/files/retrieve-contents
|
||||||
|
|
||||||
|
Example Curl
|
||||||
|
```
|
||||||
|
curl http://localhost:4000/v1/files/file-abc123/content \
|
||||||
|
-H "Authorization: Bearer sk-1234"
|
||||||
|
|
||||||
|
```
|
||||||
|
"""
|
||||||
|
from litellm.proxy.proxy_server import (
|
||||||
|
add_litellm_data_to_request,
|
||||||
|
general_settings,
|
||||||
|
get_custom_headers,
|
||||||
|
proxy_config,
|
||||||
|
proxy_logging_obj,
|
||||||
|
version,
|
||||||
|
)
|
||||||
|
|
||||||
|
data: Dict = {}
|
||||||
|
try:
|
||||||
|
|
||||||
|
# Include original request and headers in the data
|
||||||
|
data = await add_litellm_data_to_request(
|
||||||
|
data=data,
|
||||||
|
request=request,
|
||||||
|
general_settings=general_settings,
|
||||||
|
user_api_key_dict=user_api_key_dict,
|
||||||
|
version=version,
|
||||||
|
proxy_config=proxy_config,
|
||||||
|
)
|
||||||
|
|
||||||
|
if provider is None:
|
||||||
|
provider = "openai"
|
||||||
|
response = await litellm.afile_content(
|
||||||
|
custom_llm_provider=provider, file_id=file_id, **data # type: ignore
|
||||||
|
)
|
||||||
|
|
||||||
|
### ALERTING ###
|
||||||
|
asyncio.create_task(
|
||||||
|
proxy_logging_obj.update_request_status(
|
||||||
|
litellm_call_id=data.get("litellm_call_id", ""), status="success"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
### RESPONSE HEADERS ###
|
||||||
|
hidden_params = getattr(response, "_hidden_params", {}) or {}
|
||||||
|
model_id = hidden_params.get("model_id", None) or ""
|
||||||
|
cache_key = hidden_params.get("cache_key", None) or ""
|
||||||
|
api_base = hidden_params.get("api_base", None) or ""
|
||||||
|
|
||||||
|
fastapi_response.headers.update(
|
||||||
|
get_custom_headers(
|
||||||
|
user_api_key_dict=user_api_key_dict,
|
||||||
|
model_id=model_id,
|
||||||
|
cache_key=cache_key,
|
||||||
|
api_base=api_base,
|
||||||
|
version=version,
|
||||||
|
model_region=getattr(user_api_key_dict, "allowed_model_region", ""),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
httpx_response: Optional[httpx.Response] = getattr(response, "response", None)
|
||||||
|
if httpx_response is None:
|
||||||
|
raise ValueError(
|
||||||
|
f"Invalid response - response.response is None - got {response}"
|
||||||
|
)
|
||||||
|
|
||||||
|
return Response(
|
||||||
|
content=httpx_response.content,
|
||||||
|
status_code=httpx_response.status_code,
|
||||||
|
headers=httpx_response.headers,
|
||||||
|
)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
await proxy_logging_obj.post_call_failure_hook(
|
||||||
|
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
|
||||||
|
)
|
||||||
|
verbose_proxy_logger.error(
|
||||||
|
"litellm.proxy.proxy_server.retrieve_file_content(): Exception occured - {}".format(
|
||||||
|
str(e)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
verbose_proxy_logger.debug(traceback.format_exc())
|
||||||
|
if isinstance(e, HTTPException):
|
||||||
|
raise ProxyException(
|
||||||
|
message=getattr(e, "message", str(e.detail)),
|
||||||
|
type=getattr(e, "type", "None"),
|
||||||
|
param=getattr(e, "param", "None"),
|
||||||
|
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
error_msg = f"{str(e)}"
|
||||||
|
raise ProxyException(
|
||||||
|
message=getattr(e, "message", error_msg),
|
||||||
|
type=getattr(e, "type", "None"),
|
||||||
|
param=getattr(e, "param", "None"),
|
||||||
|
code=getattr(e, "status_code", 500),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.get(
|
@router.get(
|
||||||
"/{provider}/v1/files/{file_id:path}",
|
"/{provider}/v1/files/{file_id:path}",
|
||||||
dependencies=[Depends(user_api_key_auth)],
|
dependencies=[Depends(user_api_key_auth)],
|
||||||
|
@ -609,127 +734,3 @@ async def list_files(
|
||||||
param=getattr(e, "param", "None"),
|
param=getattr(e, "param", "None"),
|
||||||
code=getattr(e, "status_code", 500),
|
code=getattr(e, "status_code", 500),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.get(
|
|
||||||
"/{provider}/v1/files/{file_id:path}/content",
|
|
||||||
dependencies=[Depends(user_api_key_auth)],
|
|
||||||
tags=["files"],
|
|
||||||
)
|
|
||||||
@router.get(
|
|
||||||
"/v1/files/{file_id:path}/content",
|
|
||||||
dependencies=[Depends(user_api_key_auth)],
|
|
||||||
tags=["files"],
|
|
||||||
)
|
|
||||||
@router.get(
|
|
||||||
"/files/{file_id:path}/content",
|
|
||||||
dependencies=[Depends(user_api_key_auth)],
|
|
||||||
tags=["files"],
|
|
||||||
)
|
|
||||||
async def get_file_content(
|
|
||||||
request: Request,
|
|
||||||
fastapi_response: Response,
|
|
||||||
file_id: str,
|
|
||||||
provider: Optional[str] = None,
|
|
||||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
|
||||||
):
|
|
||||||
"""
|
|
||||||
Returns information about a specific file. that can be used across - Assistants API, Batch API
|
|
||||||
This is the equivalent of GET https://api.openai.com/v1/files/{file_id}/content
|
|
||||||
|
|
||||||
Supports Identical Params as: https://platform.openai.com/docs/api-reference/files/retrieve-contents
|
|
||||||
|
|
||||||
Example Curl
|
|
||||||
```
|
|
||||||
curl http://localhost:4000/v1/files/file-abc123/content \
|
|
||||||
-H "Authorization: Bearer sk-1234"
|
|
||||||
|
|
||||||
```
|
|
||||||
"""
|
|
||||||
from litellm.proxy.proxy_server import (
|
|
||||||
add_litellm_data_to_request,
|
|
||||||
general_settings,
|
|
||||||
get_custom_headers,
|
|
||||||
proxy_config,
|
|
||||||
proxy_logging_obj,
|
|
||||||
version,
|
|
||||||
)
|
|
||||||
|
|
||||||
data: Dict = {}
|
|
||||||
try:
|
|
||||||
|
|
||||||
# Include original request and headers in the data
|
|
||||||
data = await add_litellm_data_to_request(
|
|
||||||
data=data,
|
|
||||||
request=request,
|
|
||||||
general_settings=general_settings,
|
|
||||||
user_api_key_dict=user_api_key_dict,
|
|
||||||
version=version,
|
|
||||||
proxy_config=proxy_config,
|
|
||||||
)
|
|
||||||
|
|
||||||
if provider is None:
|
|
||||||
provider = "openai"
|
|
||||||
response = await litellm.afile_content(
|
|
||||||
custom_llm_provider=provider, file_id=file_id, **data # type: ignore
|
|
||||||
)
|
|
||||||
|
|
||||||
### ALERTING ###
|
|
||||||
asyncio.create_task(
|
|
||||||
proxy_logging_obj.update_request_status(
|
|
||||||
litellm_call_id=data.get("litellm_call_id", ""), status="success"
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
### RESPONSE HEADERS ###
|
|
||||||
hidden_params = getattr(response, "_hidden_params", {}) or {}
|
|
||||||
model_id = hidden_params.get("model_id", None) or ""
|
|
||||||
cache_key = hidden_params.get("cache_key", None) or ""
|
|
||||||
api_base = hidden_params.get("api_base", None) or ""
|
|
||||||
|
|
||||||
fastapi_response.headers.update(
|
|
||||||
get_custom_headers(
|
|
||||||
user_api_key_dict=user_api_key_dict,
|
|
||||||
model_id=model_id,
|
|
||||||
cache_key=cache_key,
|
|
||||||
api_base=api_base,
|
|
||||||
version=version,
|
|
||||||
model_region=getattr(user_api_key_dict, "allowed_model_region", ""),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
httpx_response: Optional[httpx.Response] = getattr(response, "response", None)
|
|
||||||
if httpx_response is None:
|
|
||||||
raise ValueError(
|
|
||||||
f"Invalid response - response.response is None - got {response}"
|
|
||||||
)
|
|
||||||
return Response(
|
|
||||||
content=httpx_response.content,
|
|
||||||
status_code=httpx_response.status_code,
|
|
||||||
headers=httpx_response.headers,
|
|
||||||
)
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
await proxy_logging_obj.post_call_failure_hook(
|
|
||||||
user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
|
|
||||||
)
|
|
||||||
verbose_proxy_logger.error(
|
|
||||||
"litellm.proxy.proxy_server.retrieve_file_content(): Exception occured - {}".format(
|
|
||||||
str(e)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
verbose_proxy_logger.debug(traceback.format_exc())
|
|
||||||
if isinstance(e, HTTPException):
|
|
||||||
raise ProxyException(
|
|
||||||
message=getattr(e, "message", str(e.detail)),
|
|
||||||
type=getattr(e, "type", "None"),
|
|
||||||
param=getattr(e, "param", "None"),
|
|
||||||
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
error_msg = f"{str(e)}"
|
|
||||||
raise ProxyException(
|
|
||||||
message=getattr(e, "message", error_msg),
|
|
||||||
type=getattr(e, "type", "None"),
|
|
||||||
param=getattr(e, "param", "None"),
|
|
||||||
code=getattr(e, "status_code", 500),
|
|
||||||
)
|
|
||||||
|
|
|
@ -4,3 +4,8 @@ model_list:
|
||||||
model: openai/o1-preview
|
model: openai/o1-preview
|
||||||
api_key: os.environ/OPENAI_API_KEY
|
api_key: os.environ/OPENAI_API_KEY
|
||||||
|
|
||||||
|
|
||||||
|
# for /files endpoints
|
||||||
|
files_settings:
|
||||||
|
- custom_llm_provider: openai
|
||||||
|
api_key: os.environ/OPENAI_API_KEY
|
1
tests/openai_misc_endpoints_tests/input.jsonl
Normal file
1
tests/openai_misc_endpoints_tests/input.jsonl
Normal file
|
@ -0,0 +1 @@
|
||||||
|
{"custom_id": "ae006110bb364606||/workspace/saved_models/meta-llama/Meta-Llama-3.1-8B-Instruct", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-4o-2024-05-13", "temperature": 0, "max_tokens": 1024, "response_format": {"type": "json_object"}, "messages": [{"role": "user", "content": "# Instruction \n\nYou are an expert evaluator. Your task is to evaluate the quality of the responses generated by AI models. \nWe will provide you with the user query and an AI-generated responses.\nYo must respond in json"}]}}
|
|
@ -0,0 +1,2 @@
|
||||||
|
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "my-custom-name", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
||||||
|
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "my-custom-name", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
|
1
tests/openai_misc_endpoints_tests/out.jsonl
Normal file
1
tests/openai_misc_endpoints_tests/out.jsonl
Normal file
|
@ -0,0 +1 @@
|
||||||
|
{"id": "batch_req_6765ed82629c8190b70c10c183b5e994", "custom_id": "ae006110bb364606||/workspace/saved_models/meta-llama/Meta-Llama-3.1-8B-Instruct", "response": {"status_code": 200, "request_id": "36bbc935dec50094e84af1db52cf2cc7", "body": {"id": "chatcmpl-AgfdQmdwJQ0NrQManGI8ecwMvF0ZC", "object": "chat.completion", "created": 1734733184, "model": "gpt-4o-2024-05-13", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\n \"user_query\": \"What are the benefits of using renewable energy sources?\",\n \"ai_response\": \"Renewable energy sources, such as solar, wind, and hydroelectric power, offer numerous benefits. They are sustainable and can be replenished naturally, reducing the reliance on finite fossil fuels. Additionally, renewable energy sources produce little to no greenhouse gas emissions, helping to combat climate change and reduce air pollution. They also create jobs in the renewable energy sector and can lead to energy independence for countries that invest in their development. Furthermore, renewable energy technologies often have lower operating costs once established, providing long-term economic benefits.\"\n}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 51, "completion_tokens": 128, "total_tokens": 179, "prompt_tokens_details": {"cached_tokens": 0, "audio_tokens": 0}, "completion_tokens_details": {"reasoning_tokens": 0, "audio_tokens": 0, "accepted_prediction_tokens": 0, "rejected_prediction_tokens": 0}}, "system_fingerprint": "fp_20cb129c3a"}}, "error": null}
|
|
@ -6,6 +6,9 @@ import aiohttp, openai
|
||||||
from openai import OpenAI, AsyncOpenAI
|
from openai import OpenAI, AsyncOpenAI
|
||||||
from typing import Optional, List, Union
|
from typing import Optional, List, Union
|
||||||
from test_openai_files_endpoints import upload_file, delete_file
|
from test_openai_files_endpoints import upload_file, delete_file
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
|
||||||
|
|
||||||
BASE_URL = "http://localhost:4000" # Replace with your actual base URL
|
BASE_URL = "http://localhost:4000" # Replace with your actual base URL
|
||||||
|
@ -87,6 +90,78 @@ async def test_batches_operations():
|
||||||
await delete_file(session, file_id)
|
await delete_file(session, file_id)
|
||||||
|
|
||||||
|
|
||||||
|
from openai import OpenAI
|
||||||
|
|
||||||
|
client = OpenAI(base_url=BASE_URL, api_key=API_KEY)
|
||||||
|
|
||||||
|
|
||||||
|
def create_batch_oai_sdk(filepath) -> str:
|
||||||
|
batch_input_file = client.files.create(file=open(filepath, "rb"), purpose="batch")
|
||||||
|
batch_input_file_id = batch_input_file.id
|
||||||
|
|
||||||
|
rq = client.batches.create(
|
||||||
|
input_file_id=batch_input_file_id,
|
||||||
|
endpoint="/v1/chat/completions",
|
||||||
|
completion_window="24h",
|
||||||
|
metadata={
|
||||||
|
"description": filepath,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
print(f"Batch submitted. ID: {rq.id}")
|
||||||
|
return rq.id
|
||||||
|
|
||||||
|
|
||||||
|
def await_batch_completion(batch_id: str):
|
||||||
|
while True:
|
||||||
|
batch = client.batches.retrieve(batch_id)
|
||||||
|
if batch.status == "completed":
|
||||||
|
print(f"Batch {batch_id} completed.")
|
||||||
|
return
|
||||||
|
|
||||||
|
print("waiting for batch to complete...")
|
||||||
|
time.sleep(10)
|
||||||
|
|
||||||
|
|
||||||
|
def write_content_to_file(batch_id: str, output_path: str) -> str:
|
||||||
|
batch = client.batches.retrieve(batch_id)
|
||||||
|
content = client.files.content(batch.output_file_id)
|
||||||
|
print("content from files.content", content.content)
|
||||||
|
content.write_to_file(output_path)
|
||||||
|
|
||||||
|
|
||||||
|
import jsonlines
|
||||||
|
|
||||||
|
|
||||||
|
def read_jsonl(filepath: str):
|
||||||
|
results = []
|
||||||
|
with jsonlines.open(filepath) as f:
|
||||||
|
for line in f:
|
||||||
|
results.append(line)
|
||||||
|
|
||||||
|
for item in results:
|
||||||
|
print(item)
|
||||||
|
custom_id = item["custom_id"]
|
||||||
|
print(custom_id)
|
||||||
|
|
||||||
|
|
||||||
|
def test_e2e_batches_files():
|
||||||
|
"""
|
||||||
|
[PROD Test] Ensures OpenAI Batches + files work with OpenAI SDK
|
||||||
|
"""
|
||||||
|
input_path = "input.jsonl"
|
||||||
|
output_path = "out.jsonl"
|
||||||
|
|
||||||
|
_current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
input_file_path = os.path.join(_current_dir, input_path)
|
||||||
|
output_file_path = os.path.join(_current_dir, output_path)
|
||||||
|
|
||||||
|
batch_id = create_batch_oai_sdk(input_file_path)
|
||||||
|
await_batch_completion(batch_id)
|
||||||
|
write_content_to_file(batch_id, output_file_path)
|
||||||
|
read_jsonl(output_file_path)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.skip(reason="Local only test to verify if things work well")
|
@pytest.mark.skip(reason="Local only test to verify if things work well")
|
||||||
def test_vertex_batches_endpoint():
|
def test_vertex_batches_endpoint():
|
||||||
"""
|
"""
|
|
@ -13,21 +13,27 @@ API_KEY = "sk-1234" # Replace with your actual API key
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_file_operations():
|
async def test_file_operations():
|
||||||
async with aiohttp.ClientSession() as session:
|
openai_client = AsyncOpenAI(api_key=API_KEY, base_url=BASE_URL)
|
||||||
# Test file upload and get file_id
|
file_content = b'{"prompt": "Hello", "completion": "Hi"}'
|
||||||
file_id = await upload_file(session)
|
uploaded_file = await openai_client.files.create(
|
||||||
|
purpose="fine-tune",
|
||||||
|
file=file_content,
|
||||||
|
)
|
||||||
|
list_files = await openai_client.files.list()
|
||||||
|
print("list_files=", list_files)
|
||||||
|
|
||||||
# Test list files
|
get_file = await openai_client.files.retrieve(file_id=uploaded_file.id)
|
||||||
await list_files(session)
|
print("get_file=", get_file)
|
||||||
|
|
||||||
# Test get file
|
get_file_content = await openai_client.files.content(file_id=uploaded_file.id)
|
||||||
await get_file(session, file_id)
|
print("get_file_content=", get_file_content.content)
|
||||||
|
|
||||||
# Test get file content
|
assert get_file_content.content == file_content
|
||||||
await get_file_content(session, file_id)
|
# try get_file_content.write_to_file
|
||||||
|
get_file_content.write_to_file("get_file_content.jsonl")
|
||||||
|
|
||||||
# Test delete file
|
delete_file = await openai_client.files.delete(file_id=uploaded_file.id)
|
||||||
await delete_file(session, file_id)
|
print("delete_file=", delete_file)
|
||||||
|
|
||||||
|
|
||||||
async def upload_file(session, purpose="fine-tune"):
|
async def upload_file(session, purpose="fine-tune"):
|
||||||
|
@ -81,6 +87,7 @@ async def get_file_content(session, file_id):
|
||||||
async with session.get(url, headers=headers) as response:
|
async with session.get(url, headers=headers) as response:
|
||||||
assert response.status == 200
|
assert response.status == 200
|
||||||
content = await response.text()
|
content = await response.text()
|
||||||
|
print("content from /files/{file_id}/content=", content)
|
||||||
assert content # Check if content is not empty
|
assert content # Check if content is not empty
|
||||||
print(f"Get file content successful for file ID: {file_id}")
|
print(f"Get file content successful for file ID: {file_id}")
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue