From c45e2ed48c027a2797ab6fdf7e9e2c7bbb20ff88 Mon Sep 17 00:00:00 2001
From: ishaan-jaff <ishaanjaffer0324@gmail.com>
Date: Fri, 8 Sep 2023 13:58:06 -0700
Subject: [PATCH] hosted vllm usage

---
 litellm/main.py                  |  4 ++--
 litellm/tests/test_completion.py | 26 +++++++++++++++++++++++++-
 2 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/litellm/main.py b/litellm/main.py
index 2f38ea980..96dd50341 100644
--- a/litellm/main.py
+++ b/litellm/main.py
@@ -346,9 +346,9 @@ def completion(
             ## RESPONSE OBJECT
             completion_response = response["choices"][0]["text"]
             model_response["choices"][0]["message"]["content"] = completion_response
-            model_response["created"] = response["created"]
+            model_response["created"] = response.get("created", time.time())
             model_response["model"] = model
-            model_response["usage"] = response["usage"]
+            model_response["usage"] = response.get("usage", 0)
             response = model_response
         elif (
             "replicate" in model or 
diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py
index 4f17eaa20..240a89a55 100644
--- a/litellm/tests/test_completion.py
+++ b/litellm/tests/test_completion.py
@@ -14,7 +14,6 @@ from litellm import embedding, completion, text_completion, completion_cost
 
 litellm.vertex_project = "pathrise-convert-1606954137718"
 litellm.vertex_location = "us-central1"
-litellm.use_client = True
 
 user_message = "Write a short poem about the sky"
 messages = [{"content": user_message, "role": "user"}]
@@ -482,6 +481,31 @@ def test_completion_sagemaker():
 
 # test_completion_vllm()
 
+# def test_completion_hosted_vllm():
+#     # this tests calling a server where vllm is hosted
+#     # this should make an openai.Completion() call to the specified api_base
+#     # send a request to this proxy server: https://replit.com/@BerriAI/openai-proxy#main.py
+#     # it checks if model == facebook/opt-125m and returns test passed
+#     try:
+#         # litellm.set_verbose = True
+#         response = completion(
+#             model="facebook/opt-125m", 
+#             messages=messages,
+#             temperature=0.2,
+#             max_tokens=80,
+#             api_base="https://openai-proxy.berriai.repl.co/v1",
+#             custom_llm_provider="openai"
+#         )
+#         print(response)
+
+#         if response['choices'][0]['message']['content'] != "passed":
+#             # see https://replit.com/@BerriAI/openai-proxy#main.py
+#             pytest.fail(f"Error occurred: proxy server did not respond")
+#     except Exception as e:
+#         pytest.fail(f"Error occurred: {e}")
+
+# test_completion_hosted_vllm()
+
 # def test_completion_custom_api_base():
 #     try:
 #         response = completion(